02:59:31.476 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:59:31.476 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskKOuND7/test-tmp
02:59:31.476 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
02:59:31.477 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskKOuND7/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
02:59:31.479 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
02:59:31.824 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:59:31.824 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:59:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:31.826989 20271 mini_oidc.cc:150] Starting JWKS server
02:59:31.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:31.829264 20271 webserver.cc:577] Webserver started at http://127.0.0.1:40665/ using document root <none> and password file <none>
02:59:31.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:31.829501 20271 mini_oidc.cc:186] Starting OIDC Discovery server
02:59:31.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:31.829982 20271 webserver.cc:577] Webserver started at http://127.0.0.1:36533/ using document root <none> and password file <none>
02:59:31.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-21T02:59:31Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:59:31.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-21T02:59:31Z Disabled control of system clock
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:31.858999 20271 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskKOuND7/build/asan/bin/kudu
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKOuND7/build/asan/bin/kudu
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-0/wal
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-0/data
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-0/logs
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-0/data/info.pb
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.203.254:46111
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.203.254
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.203.212:34207
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.203.254:46111,127.19.203.253:34409,127.19.203.252:40563
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40665/jwks.json
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
02:59:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
02:59:32.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.203145 20282 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:59:32.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.203560 20282 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:59:32.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.203706 20282 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:59:32.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.212438 20282 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:59:32.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.212615 20282 flags.cc:432] Enabled experimental flag: --txn_manager_enabled=true
02:59:32.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.212692 20282 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:59:32.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.212764 20282 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:59:32.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.212879 20282 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:59:32.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.212991 20282 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:59:32.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.213109 20282 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:40665/jwks.json
02:59:32.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.223513 20282 master_runner.cc:387] Master server non-default flags:
02:59:32.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-0/logs
02:59:32.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:59:32.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:59:32.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:59:32.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.203.212:34207
02:59:32.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:59:32.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-0/data
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-0/wal
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.203.254:46111,127.19.203.253:34409,127.19.203.252:40563
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.203.254:46111
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40665/jwks.json
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-0/data/info.pb
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.203.254
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 05464f8854a90597ddf8dfd45c2061c63b0b716b
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Nov 2026 02:43:19 UTC on bdcb31816ec0
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14886
02:59:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
02:59:32.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.225391 20282 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:59:32.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.227267 20282 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:59:32.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.235925 20287 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:59:32.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.236465 20290 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:59:32.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.236580 20288 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:59:32.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.237648 20282 server_base.cc:1061] running on GCE node
02:59:32.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.238498 20282 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:59:32.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.240043 20282 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:59:32.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.241302 20282 hybrid_clock.cc:648] HybridClock initialized: now 1795229972241220 us; error 51 us; skew 500 ppm
02:59:32.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.241659 20282 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:59:32.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.243914 20282 webserver.cc:577] Webserver started at http://127.19.203.254:34241/ using document root <none> and password file <none>
02:59:32.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.244637 20282 fs_manager.cc:362] Metadata directory not provided
02:59:32.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.244812 20282 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:59:32.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.245245 20282 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:59:32.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.247762 20282 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-0/data/instance:
02:59:32.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "56099a6022da440a81f20e6fe8903fb0"
02:59:32.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-21 02:59:32 on dist-test-slave-mbqs"
02:59:32.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.248629 20282 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-0/wal/instance:
02:59:32.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "56099a6022da440a81f20e6fe8903fb0"
02:59:32.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-21 02:59:32 on dist-test-slave-mbqs"
02:59:32.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.254278 20282 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.001s sys 0.003s
02:59:32.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.257792 20296 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:59:32.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.259213 20282 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
02:59:32.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.259413 20282 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-0/data,/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-0/wal
02:59:32.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "56099a6022da440a81f20e6fe8903fb0"
02:59:32.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-21 02:59:32 on dist-test-slave-mbqs"
02:59:32.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.259616 20282 fs_report.cc:389] FS layout report
02:59:32.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:59:32.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-0/wal
02:59:32.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-0/wal
02:59:32.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-0/data/data
02:59:32.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:59:32.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:59:32.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:59:32.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:59:32.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:59:32.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:59:32.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:59:32.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:59:32.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:59:32.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:59:32.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:59:32.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.277515 20282 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:59:32.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.278471 20282 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:59:32.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.278818 20282 kserver.cc:163] Server-wide thread pool size limit: 3276
02:59:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.303063 20282 rpc_server.cc:307] RPC server started. Bound to: 127.19.203.254:46111
02:59:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.303094 20357 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.203.254:46111 every 8 connection(s)
02:59:32.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.304682 20282 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-0/data/info.pb
02:59:32.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.306252 20271 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskKOuND7/build/asan/bin/kudu as pid 20282
02:59:32.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.306571 20271 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-0/wal/instance
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.309001 20271 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskKOuND7/build/asan/bin/kudu
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKOuND7/build/asan/bin/kudu
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-1/wal
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-1/data
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-1/logs
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-1/data/info.pb
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.203.253:34409
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.203.253
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:59:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.203.212:34207
02:59:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:59:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:59:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:59:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:59:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.203.254:46111,127.19.203.253:34409,127.19.203.252:40563
02:59:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:59:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40665/jwks.json
02:59:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
02:59:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
02:59:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.309516 20358 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:59:32.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.316447 20358 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } has no permanent_uuid. Determining permanent_uuid...
02:59:32.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.338351 20358 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } has no permanent_uuid. Determining permanent_uuid...
02:59:32.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.339736 20299 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.203.253:34409: connect: Connection refused (error 111)
02:59:32.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.341536 20358 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.203.253:34409: Network error: Client connection negotiation failed: client connection to 127.19.203.253:34409: connect: Connection refused (error 111)
02:59:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.390885 20358 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } attempt: 1
02:59:32.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.393535 20358 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.203.253:34409: Network error: Client connection negotiation failed: client connection to 127.19.203.253:34409: connect: Connection refused (error 111)
02:59:32.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.461831 20358 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } attempt: 2
02:59:32.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.463994 20358 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.203.253:34409: Network error: Client connection negotiation failed: client connection to 127.19.203.253:34409: connect: Connection refused (error 111)
02:59:32.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.555289 20358 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } attempt: 3
02:59:32.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.557664 20358 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.203.253:34409: Network error: Client connection negotiation failed: client connection to 127.19.203.253:34409: connect: Connection refused (error 111)
02:59:32.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.667855 20360 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:59:32.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.668176 20360 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:59:32.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.668263 20360 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:59:32.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.677086 20360 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:59:32.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.677189 20360 flags.cc:432] Enabled experimental flag: --txn_manager_enabled=true
02:59:32.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.677242 20360 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:59:32.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.677289 20360 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:59:32.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.677359 20360 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:59:32.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.677410 20360 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:59:32.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.677459 20360 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:40665/jwks.json
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.687987 20360 master_runner.cc:387] Master server non-default flags:
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-1/logs
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.203.212:34207
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-1/data
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-1/wal
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.203.254:46111,127.19.203.253:34409,127.19.203.252:40563
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.203.253:34409
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40665/jwks.json
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-1/data/info.pb
02:59:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.203.253
02:59:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:59:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:59:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:59:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:59:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:59:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:59:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:59:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:59:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:59:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 05464f8854a90597ddf8dfd45c2061c63b0b716b
02:59:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:59:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Nov 2026 02:43:19 UTC on bdcb31816ec0
02:59:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14886
02:59:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
02:59:32.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.689859 20360 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:59:32.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.691723 20360 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:59:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.700453 20369 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:59:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.700484 20368 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:59:32.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.700922 20358 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } attempt: 4
02:59:32.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.701856 20371 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:59:32.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.702787 20360 server_base.cc:1061] running on GCE node
02:59:32.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.703382 20358 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.203.253:34409: Network error: Client connection negotiation failed: client connection to 127.19.203.253:34409: connect: Connection refused (error 111)
02:59:32.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.703730 20360 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:59:32.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.705137 20360 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:59:32.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.706422 20360 hybrid_clock.cc:648] HybridClock initialized: now 1795229972706312 us; error 56 us; skew 500 ppm
02:59:32.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.706775 20360 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:59:32.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.709214 20360 webserver.cc:577] Webserver started at http://127.19.203.253:35309/ using document root <none> and password file <none>
02:59:32.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.709969 20360 fs_manager.cc:362] Metadata directory not provided
02:59:32.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.710119 20360 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:59:32.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.710544 20360 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:59:32.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.713104 20360 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-1/data/instance:
02:59:32.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1e3c2fda688743cfb0f78de4c81a953c"
02:59:32.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-21 02:59:32 on dist-test-slave-mbqs"
02:59:32.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.713955 20360 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-1/wal/instance:
02:59:32.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1e3c2fda688743cfb0f78de4c81a953c"
02:59:32.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-21 02:59:32 on dist-test-slave-mbqs"
02:59:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.719584 20360 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
02:59:32.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.723301 20377 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:59:32.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.724720 20360 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
02:59:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.724944 20360 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-1/data,/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-1/wal
02:59:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1e3c2fda688743cfb0f78de4c81a953c"
02:59:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-21 02:59:32 on dist-test-slave-mbqs"
02:59:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.725128 20360 fs_report.cc:389] FS layout report
02:59:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:59:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-1/wal
02:59:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-1/wal
02:59:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-1/data/data
02:59:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:59:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:59:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:59:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:59:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:59:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:59:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:59:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:59:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:59:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:59:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:59:32.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.759413 20360 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:59:32.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.760286 20360 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:59:32.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.760547 20360 kserver.cc:163] Server-wide thread pool size limit: 3276
02:59:32.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.786437 20360 rpc_server.cc:307] RPC server started. Bound to: 127.19.203.253:34409
02:59:32.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.786441 20438 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.203.253:34409 every 8 connection(s)
02:59:32.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.788166 20360 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-1/data/info.pb
02:59:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.789196 20271 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskKOuND7/build/asan/bin/kudu as pid 20360
02:59:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.789387 20271 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-1/wal/instance
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.790432 20271 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskKOuND7/build/asan/bin/kudu
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKOuND7/build/asan/bin/kudu
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-2/wal
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-2/data
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-2/logs
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-2/data/info.pb
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.203.252:40563
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.203.252
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.203.212:34207
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.203.254:46111,127.19.203.253:34409,127.19.203.252:40563
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40665/jwks.json
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
02:59:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.792963 20439 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:59:32.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.802250 20439 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } has no permanent_uuid. Determining permanent_uuid...
02:59:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.817734 20439 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } has no permanent_uuid. Determining permanent_uuid...
02:59:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.827781 20439 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } has no permanent_uuid. Determining permanent_uuid...
02:59:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.829094 20381 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:32.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.830951 20439 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.203.252:40563: Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:32.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.880280 20439 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } attempt: 1
02:59:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.882570 20439 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.203.252:40563: Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:32.950871 20439 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } attempt: 2
02:59:32.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:32.953117 20439 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.203.252:40563: Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:33.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.002668 20358 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } attempt: 5
02:59:33.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.010288 20358 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } has no permanent_uuid. Determining permanent_uuid...
02:59:33.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.012318 20358 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.203.252:40563: Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:33.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.044407 20439 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } attempt: 3
02:59:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.046597 20439 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.203.252:40563: Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:33.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.063552 20358 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } attempt: 1
02:59:33.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.065737 20358 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.203.252:40563: Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:33.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.134096 20358 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } attempt: 2
02:59:33.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.136871 20358 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.203.252:40563: Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:33.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.156445 20441 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:59:33.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.156881 20441 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:59:33.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.156978 20441 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:59:33.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.166018 20441 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:59:33.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.166126 20441 flags.cc:432] Enabled experimental flag: --txn_manager_enabled=true
02:59:33.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.166175 20441 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:59:33.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.166217 20441 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:59:33.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.166275 20441 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:59:33.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.166320 20441 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:59:33.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.166364 20441 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:40665/jwks.json
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.176986 20441 master_runner.cc:387] Master server non-default flags:
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-2/logs
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.203.212:34207
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-2/data
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-2/wal
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.203.254:46111,127.19.203.253:34409,127.19.203.252:40563
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.203.252:40563
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40665/jwks.json
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-2/data/info.pb
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.203.252
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 05464f8854a90597ddf8dfd45c2061c63b0b716b
02:59:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:59:33.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Nov 2026 02:43:19 UTC on bdcb31816ec0
02:59:33.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14886
02:59:33.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
02:59:33.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.178704 20441 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:59:33.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.180498 20441 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:59:33.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.189637 20449 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:59:33.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.189713 20450 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:59:33.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.189824 20439 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } attempt: 4
02:59:33.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.189684 20452 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:59:33.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.191361 20441 server_base.cc:1061] running on GCE node
02:59:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.192480 20441 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:59:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.192852 20439 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.203.252:40563: Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:33.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.194303 20441 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:59:33.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.195564 20441 hybrid_clock.cc:648] HybridClock initialized: now 1795229973195499 us; error 37 us; skew 500 ppm
02:59:33.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.195899 20441 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:59:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.198601 20441 webserver.cc:577] Webserver started at http://127.19.203.252:35205/ using document root <none> and password file <none>
02:59:33.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.199337 20441 fs_manager.cc:362] Metadata directory not provided
02:59:33.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.199447 20441 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:59:33.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.199777 20441 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:59:33.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.202389 20441 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-2/data/instance:
02:59:33.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "efb4c69b20bd4f9692d496c33ddbb24e"
02:59:33.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-21 02:59:33 on dist-test-slave-mbqs"
02:59:33.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.203178 20441 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-2/wal/instance:
02:59:33.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "efb4c69b20bd4f9692d496c33ddbb24e"
02:59:33.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-21 02:59:33 on dist-test-slave-mbqs"
02:59:33.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.209405 20441 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
02:59:33.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.213744 20458 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:59:33.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.215456 20441 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
02:59:33.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.215658 20441 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-2/data,/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-2/wal
02:59:33.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "efb4c69b20bd4f9692d496c33ddbb24e"
02:59:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-21 02:59:33 on dist-test-slave-mbqs"
02:59:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.215860 20441 fs_report.cc:389] FS layout report
02:59:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:59:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-2/wal
02:59:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-2/wal
02:59:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-2/data/data
02:59:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:59:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:59:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:59:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:59:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:59:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:59:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:59:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:59:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:59:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:59:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:59:33.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.243201 20358 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } attempt: 3
02:59:33.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.243541 20441 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:59:33.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.244576 20441 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:59:33.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.244997 20441 kserver.cc:163] Server-wide thread pool size limit: 3276
02:59:33.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.245632 20358 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.203.252:40563: Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:33.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.274326 20441 rpc_server.cc:307] RPC server started. Bound to: 127.19.203.252:40563
02:59:33.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.274359 20519 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.203.252:40563 every 8 connection(s)
02:59:33.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.276073 20441 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-2/data/info.pb
02:59:33.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.280947 20520 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:59:33.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.281930 20271 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskKOuND7/build/asan/bin/kudu as pid 20441
02:59:33.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.282177 20271 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/master-2/wal/instance
02:59:33.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.288625 20520 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } has no permanent_uuid. Determining permanent_uuid...
02:59:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.303144 20520 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } has no permanent_uuid. Determining permanent_uuid...
02:59:33.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.312589 20520 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } has no permanent_uuid. Determining permanent_uuid...
02:59:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.327687 20520 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e: Bootstrap starting.
02:59:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.331821 20520 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e: Neither blocks nor log segments found. Creating new log.
02:59:33.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.333108 20520 log.cc:826] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e: Log is configured to *not* fsync() on all Append() calls
02:59:33.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.336174 20520 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e: No bootstrap required, opened a new log
02:59:33.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.341919 20520 raft_consensus.cc:366] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:33.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.342195 20520 raft_consensus.cc:392] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:59:33.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.342278 20520 raft_consensus.cc:787] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: efb4c69b20bd4f9692d496c33ddbb24e, State: Initialized, Role: FOLLOWER
02:59:33.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.342785 20520 consensus_queue.cc:261] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:33.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.343632 20529 sys_catalog.cc:455] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } } }
02:59:33.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.343915 20529 sys_catalog.cc:458] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [sys.catalog]: This master's current role is: FOLLOWER
02:59:33.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.344547 20520 sys_catalog.cc:565] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [sys.catalog]: configured and running, proceeding with master startup.
02:59:33.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.344993 20529 raft_consensus.cc:534] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:59:33.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.345173 20529 raft_consensus.cc:556] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:33.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.347877 20529 leader_election.cc:290] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111), 1e3c2fda688743cfb0f78de4c81a953c (127.19.203.253:34409)
02:59:33.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.357091 20540 catalog_manager.cc:1609] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:59:33.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.357336 20540 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:59:33.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.353762 20441 master_runner.cc:187] Error getting master registration for 127.19.203.254:46111: OK, instance_id { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" instance_seqno: 1795229972297304 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:59:33.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.361122 20441 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:59:33.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.422868 20358 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } attempt: 4
02:59:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.436193 20358 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0: Bootstrap starting.
02:59:33.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.440629 20358 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0: Neither blocks nor log segments found. Creating new log.
02:59:33.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.443010 20358 log.cc:826] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0: Log is configured to *not* fsync() on all Append() calls
02:59:33.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.446360 20358 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0: No bootstrap required, opened a new log
02:59:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.454283 20358 raft_consensus.cc:366] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.454634 20358 raft_consensus.cc:392] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:59:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.454746 20358 raft_consensus.cc:787] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56099a6022da440a81f20e6fe8903fb0, State: Initialized, Role: FOLLOWER
02:59:33.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.455472 20358 consensus_queue.cc:261] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:33.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.456562 20543 sys_catalog.cc:455] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } } }
02:59:33.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.457000 20543 sys_catalog.cc:458] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [sys.catalog]: This master's current role is: FOLLOWER
02:59:33.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.458592 20358 sys_catalog.cc:565] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [sys.catalog]: configured and running, proceeding with master startup.
02:59:33.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.458732 20323 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56099a6022da440a81f20e6fe8903fb0" is_pre_election: true
02:59:33.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.459229 20323 raft_consensus.cc:2062] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate efb4c69b20bd4f9692d496c33ddbb24e in term 0
02:59:33.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.459867 20459 leader_election.cc:304] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 56099a6022da440a81f20e6fe8903fb0, efb4c69b20bd4f9692d496c33ddbb24e; no voters:
02:59:33.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.460389 20529 raft_consensus.cc:3024] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [term 0 FOLLOWER]: Leader pre-election won for term 1
02:59:33.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.460572 20529 raft_consensus.cc:534] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:59:33.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.460700 20529 raft_consensus.cc:3296] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [term 0 FOLLOWER]: Advancing to term 1
02:59:33.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.476529 20529 raft_consensus.cc:556] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:33.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.480252 20323 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56099a6022da440a81f20e6fe8903fb0"
02:59:33.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.480515 20323 raft_consensus.cc:3296] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 0 FOLLOWER]: Advancing to term 1
02:59:33.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.480901 20529 leader_election.cc:290] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [CANDIDATE]: Term 1 election: Requested vote from peers 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111), 1e3c2fda688743cfb0f78de4c81a953c (127.19.203.253:34409)
02:59:33.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.481302 20323 raft_consensus.cc:2062] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate efb4c69b20bd4f9692d496c33ddbb24e in term 1
02:59:33.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.485337 20459 leader_election.cc:304] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 56099a6022da440a81f20e6fe8903fb0, efb4c69b20bd4f9692d496c33ddbb24e; no voters:
02:59:33.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.485700 20529 raft_consensus.cc:3024] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [term 1 FOLLOWER]: Leader election won for term 1
02:59:33.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.485874 20529 raft_consensus.cc:744] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [term 1 LEADER]: Becoming Leader. State: Replica: efb4c69b20bd4f9692d496c33ddbb24e, State: Running, Role: LEADER
02:59:33.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.486479 20529 consensus_queue.cc:238] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:33.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.492055 20439 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } attempt: 5
02:59:33.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.495231 20546 sys_catalog.cc:455] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [sys.catalog]: SysCatalogTable state changed. Reason: New leader efb4c69b20bd4f9692d496c33ddbb24e. Latest consensus state: current_term: 1 leader_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } } }
02:59:33.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.495697 20546 sys_catalog.cc:458] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [sys.catalog]: This master's current role is: LEADER
02:59:33.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.497537 20464 tablet.cc:2554] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:59:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.508831 20554 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
02:59:33.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.511664 20554 catalog_manager.cc:1535] Initializing Kudu cluster ID...
02:59:33.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.514231 20553 consensus_queue.cc:1049] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [LEADER]: Connected to new peer: Peer: permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:59:33.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.515031 20439 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c: Bootstrap starting.
02:59:33.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.531926 20302 tablet.cc:2554] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:59:33.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.532135 20543 sys_catalog.cc:455] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader efb4c69b20bd4f9692d496c33ddbb24e. Latest consensus state: current_term: 1 leader_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } } }
02:59:33.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.532408 20543 sys_catalog.cc:458] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [sys.catalog]: This master's current role is: FOLLOWER
02:59:33.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.532294 20439 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c: Neither blocks nor log segments found. Creating new log.
02:59:33.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.542264 20439 log.cc:826] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c: Log is configured to *not* fsync() on all Append() calls
02:59:33.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.542418 20529 sys_catalog.cc:455] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } } }
02:59:33.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.542682 20529 sys_catalog.cc:458] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [sys.catalog]: This master's current role is: LEADER
02:59:33.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.544925 20282 master_runner.cc:187] Error getting master registration for 127.19.203.253:34409: OK, instance_id { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" instance_seqno: 1795229972780647 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:59:33.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.551755 20282 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:59:33.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.557027 20439 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c: No bootstrap required, opened a new log
02:59:33.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.557662 20560 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:59:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:33.557971 20560 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:59:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.560513 20543 sys_catalog.cc:455] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } } }
02:59:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.560701 20543 sys_catalog.cc:458] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [sys.catalog]: This master's current role is: FOLLOWER
02:59:33.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.568303 20439 raft_consensus.cc:366] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:33.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.568660 20439 raft_consensus.cc:392] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:59:33.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.568810 20439 raft_consensus.cc:787] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1e3c2fda688743cfb0f78de4c81a953c, State: Initialized, Role: FOLLOWER
02:59:33.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.569752 20439 consensus_queue.cc:261] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:33.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.571640 20554 catalog_manager.cc:1398] Generated new cluster ID: c9d1ae9ce84b4ca9b3bcaf9750cc5c21
02:59:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.571779 20554 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
02:59:33.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.604508 20567 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } } }
02:59:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.620894 20567 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [sys.catalog]: This master's current role is: FOLLOWER
02:59:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.621335 20567 raft_consensus.cc:534] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:59:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.621620 20567 raft_consensus.cc:556] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:33.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.624249 20322 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1e3c2fda688743cfb0f78de4c81a953c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56099a6022da440a81f20e6fe8903fb0" is_pre_election: true
02:59:33.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.624349 20567 leader_election.cc:290] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
02:59:33.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.625859 20439 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [sys.catalog]: configured and running, proceeding with master startup.
02:59:33.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.625886 20485 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1e3c2fda688743cfb0f78de4c81a953c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" is_pre_election: true
02:59:33.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.626971 20404 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1e3c2fda688743cfb0f78de4c81a953c" is_pre_election: true
02:59:33.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.627471 20404 raft_consensus.cc:2062] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate efb4c69b20bd4f9692d496c33ddbb24e in term 0
02:59:33.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.628839 20403 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1e3c2fda688743cfb0f78de4c81a953c"
02:59:33.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.633422 20402 raft_consensus.cc:3296] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 0 FOLLOWER]: Advancing to term 1
02:59:33.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.634541 20381 leader_election.cc:304] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c; no voters: 56099a6022da440a81f20e6fe8903fb0, efb4c69b20bd4f9692d496c33ddbb24e
02:59:33.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.636644 20567 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:59:33.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.637323 20553 consensus_queue.cc:1049] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:59:33.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.676460 20554 catalog_manager.cc:1421] Generated new certificate authority record
02:59:33.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.678332 20554 catalog_manager.cc:1555] Loading token signing keys...
02:59:33.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.690198 20566 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } } }
02:59:33.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.690580 20566 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [sys.catalog]: This master's current role is: FOLLOWER
02:59:33.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.690929 20529 sys_catalog.cc:455] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } } }
02:59:33.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.691154 20529 sys_catalog.cc:458] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e [sys.catalog]: This master's current role is: LEADER
02:59:33.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.691883 20567 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [sys.catalog]: SysCatalogTable state changed. Reason: New leader efb4c69b20bd4f9692d496c33ddbb24e. Latest consensus state: current_term: 1 leader_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } } }
02:59:33.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.692138 20567 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [sys.catalog]: This master's current role is: FOLLOWER
02:59:33.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.713121 20554 catalog_manager.cc:6237] T 00000000000000000000000000000000 P efb4c69b20bd4f9692d496c33ddbb24e: Generated new TSK 0
02:59:33.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.714219 20554 catalog_manager.cc:1565] Initializing in-progress tserver states...
02:59:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.731277 20271 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskKOuND7/build/asan/bin/kudu
02:59:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKOuND7/build/asan/bin/kudu
02:59:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-0/wal
02:59:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-0/data
02:59:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:59:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:59:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:59:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:59:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:59:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:59:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-0/logs
02:59:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-0/data/info.pb
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.203.193:0
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.203.193
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.203.193
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.203.254:46111,127.19.203.253:34409,127.19.203.252:40563
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.203.212:34207
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
02:59:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
02:59:33.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.805981 20583 catalog_manager.cc:1310] Loaded cluster ID: c9d1ae9ce84b4ca9b3bcaf9750cc5c21
02:59:33.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.806097 20583 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c: loading cluster ID for follower catalog manager: success
02:59:33.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.809973 20583 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c: acquiring CA information for follower catalog manager: success
02:59:33.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:33.812556 20583 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:59:34.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.189738 20579 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:59:34.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.190142 20579 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:59:34.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.190363 20579 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:59:34.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.200378 20579 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:59:34.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.200616 20579 flags.cc:432] Enabled experimental flag: --txn_keepalive_interval_ms=1000
02:59:34.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.200706 20579 flags.cc:432] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
02:59:34.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.200836 20579 flags.cc:432] Enabled experimental flag: --enable_txn_system_client_init=true
02:59:34.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.201004 20579 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.203.193
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.213032 20579 tablet_server_runner.cc:78] Tablet server non-default flags:
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-0/logs
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.203.212:34207
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-0/data
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-0/wal
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.203.193:0
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-0/data/info.pb
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.203.193
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.203.254:46111,127.19.203.253:34409,127.19.203.252:40563
02:59:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:59:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:59:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:59:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:59:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:59:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.203.193
02:59:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:59:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:59:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:59:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 05464f8854a90597ddf8dfd45c2061c63b0b716b
02:59:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:59:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Nov 2026 02:43:19 UTC on bdcb31816ec0
02:59:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14886
02:59:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
02:59:34.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.215294 20579 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:59:34.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.217788 20579 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:59:34.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.232420 20589 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:59:34.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.234617 20590 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:59:34.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.241940 20592 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:59:34.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.244999 20579 server_base.cc:1061] running on GCE node
02:59:34.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.245816 20579 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:59:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.247025 20579 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:59:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.248426 20579 hybrid_clock.cc:648] HybridClock initialized: now 1795229974248301 us; error 94 us; skew 500 ppm
02:59:34.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.248874 20579 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:59:34.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.251781 20579 webserver.cc:577] Webserver started at http://127.19.203.193:46849/ using document root <none> and password file <none>
02:59:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.252535 20579 fs_manager.cc:362] Metadata directory not provided
02:59:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.252681 20579 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:59:34.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.253113 20579 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:59:34.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.255600 20579 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-0/data/instance:
02:59:34.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1224c25229164dc7bda79147397a905c"
02:59:34.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-21 02:59:34 on dist-test-slave-mbqs"
02:59:34.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.256521 20579 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-0/wal/instance:
02:59:34.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1224c25229164dc7bda79147397a905c"
02:59:34.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-21 02:59:34 on dist-test-slave-mbqs"
02:59:34.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.263073 20579 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
02:59:34.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.267179 20598 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:59:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.269001 20579 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:59:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.269222 20579 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-0/data,/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-0/wal
02:59:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1224c25229164dc7bda79147397a905c"
02:59:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-21 02:59:34 on dist-test-slave-mbqs"
02:59:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.269474 20579 fs_report.cc:389] FS layout report
02:59:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:59:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-0/wal
02:59:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-0/wal
02:59:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-0/data/data
02:59:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:59:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:59:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:59:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:59:34.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:59:34.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:59:34.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:59:34.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:59:34.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:59:34.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:59:34.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:59:34.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.310189 20579 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:59:34.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.311412 20579 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:59:34.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.311760 20579 kserver.cc:163] Server-wide thread pool size limit: 3276
02:59:34.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.325757 20579 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:59:34.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.326102 20579 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
02:59:34.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.326356 20579 ts_tablet_manager.cc:662] Registered 0 tablets
02:59:34.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.326546 20579 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:59:34.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.425508 20579 rpc_server.cc:307] RPC server started. Bound to: 127.19.203.193:37543
02:59:34.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.425563 20722 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.203.193:37543 every 8 connection(s)
02:59:34.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.427050 20579 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-0/data/info.pb
02:59:34.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.429922 20724 heartbeater.cc:345] Connected to a master server at 127.19.203.254:46111
02:59:34.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.430335 20724 heartbeater.cc:462] Registering TS with master...
02:59:34.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.431476 20724 heartbeater.cc:508] Master 127.19.203.254:46111 requested a full tablet report, sending...
02:59:34.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.434645 20313 ts_manager.cc:210] Registered new tserver with Master: 1224c25229164dc7bda79147397a905c (127.19.203.193:37543)
02:59:34.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.435160 20271 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskKOuND7/build/asan/bin/kudu as pid 20579
02:59:34.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.435353 20271 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-0/wal/instance
02:59:34.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.439260 20723 heartbeater.cc:345] Connected to a master server at 127.19.203.252:40563
02:59:34.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.439523 20723 heartbeater.cc:462] Registering TS with master...
02:59:34.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.440109 20723 heartbeater.cc:508] Master 127.19.203.252:40563 requested a full tablet report, sending...
02:59:34.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.440290 20271 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskKOuND7/build/asan/bin/kudu
02:59:34.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKOuND7/build/asan/bin/kudu
02:59:34.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-1/wal
02:59:34.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-1/data
02:59:34.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:59:34.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:59:34.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:59:34.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:59:34.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-1/logs
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-1/data/info.pb
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.203.194:0
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.203.194
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.203.194
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.203.254:46111,127.19.203.253:34409,127.19.203.252:40563
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.203.212:34207
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
02:59:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
02:59:34.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.443127 20475 ts_manager.cc:210] Registered new tserver with Master: 1224c25229164dc7bda79147397a905c (127.19.203.193:37543)
02:59:34.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.445372 20475 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.19.203.193:40455
02:59:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.448833 20725 heartbeater.cc:345] Connected to a master server at 127.19.203.253:34409
02:59:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.449177 20725 heartbeater.cc:462] Registering TS with master...
02:59:34.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.450194 20725 heartbeater.cc:508] Master 127.19.203.253:34409 requested a full tablet report, sending...
02:59:34.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.452152 20393 ts_manager.cc:210] Registered new tserver with Master: 1224c25229164dc7bda79147397a905c (127.19.203.193:37543)
02:59:34.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.560212 20560 catalog_manager.cc:1310] Loaded cluster ID: c9d1ae9ce84b4ca9b3bcaf9750cc5c21
02:59:34.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.560343 20560 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0: loading cluster ID for follower catalog manager: success
02:59:34.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.563066 20560 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0: acquiring CA information for follower catalog manager: success
02:59:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.565096 20560 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:59:34.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.793704 20728 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:59:34.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.794073 20728 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:59:34.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.794224 20728 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:59:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.803180 20728 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:59:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.803386 20728 flags.cc:432] Enabled experimental flag: --txn_keepalive_interval_ms=1000
02:59:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.803468 20728 flags.cc:432] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
02:59:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.803529 20728 flags.cc:432] Enabled experimental flag: --enable_txn_system_client_init=true
02:59:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.803684 20728 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.203.194
02:59:34.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.815148 20728 tablet_server_runner.cc:78] Tablet server non-default flags:
02:59:34.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-1/logs
02:59:34.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:59:34.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:59:34.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:59:34.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.203.212:34207
02:59:34.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:59:34.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:59:34.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-1/data
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-1/wal
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.203.194:0
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-1/data/info.pb
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.203.194
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.203.254:46111,127.19.203.253:34409,127.19.203.252:40563
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.203.194
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 05464f8854a90597ddf8dfd45c2061c63b0b716b
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Nov 2026 02:43:19 UTC on bdcb31816ec0
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14886
02:59:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
02:59:34.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.817302 20728 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:59:34.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.819286 20728 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:59:34.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.830897 20733 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:59:34.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.830873 20736 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:59:34.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:34.831146 20734 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:59:34.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.832261 20728 server_base.cc:1061] running on GCE node
02:59:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.832932 20728 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:59:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.833835 20728 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:59:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.835062 20728 hybrid_clock.cc:648] HybridClock initialized: now 1795229974834989 us; error 74 us; skew 500 ppm
02:59:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.835330 20728 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:59:34.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.837551 20728 webserver.cc:577] Webserver started at http://127.19.203.194:32833/ using document root <none> and password file <none>
02:59:34.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.838244 20728 fs_manager.cc:362] Metadata directory not provided
02:59:34.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.838342 20728 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:59:34.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.838649 20728 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:59:34.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.841174 20728 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-1/data/instance:
02:59:34.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "decd50576be846f1a20a1684c18ab77f"
02:59:34.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-21 02:59:34 on dist-test-slave-mbqs"
02:59:34.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.841904 20728 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-1/wal/instance:
02:59:34.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "decd50576be846f1a20a1684c18ab77f"
02:59:34.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-21 02:59:34 on dist-test-slave-mbqs"
02:59:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.847205 20728 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
02:59:34.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.850644 20742 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:59:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.852146 20728 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
02:59:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.852293 20728 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-1/data,/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-1/wal
02:59:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "decd50576be846f1a20a1684c18ab77f"
02:59:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-21 02:59:34 on dist-test-slave-mbqs"
02:59:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.852453 20728 fs_report.cc:389] FS layout report
02:59:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:59:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-1/wal
02:59:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-1/wal
02:59:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-1/data/data
02:59:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:59:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:59:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:59:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:59:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:59:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:59:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:59:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:59:34.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:59:34.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:59:34.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:59:34.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.867201 20728 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:59:34.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.868028 20728 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:59:34.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.868283 20728 kserver.cc:163] Server-wide thread pool size limit: 3276
02:59:34.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.871325 20728 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:59:34.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.871459 20728 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:59:34.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.871541 20728 ts_tablet_manager.cc:662] Registered 0 tablets
02:59:34.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.871593 20728 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:59:34.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.947916 20728 rpc_server.cc:307] RPC server started. Bound to: 127.19.203.194:41089
02:59:34.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.948036 20863 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.203.194:41089 every 8 connection(s)
02:59:34.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.949366 20728 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-1/data/info.pb
02:59:34.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.952157 20865 heartbeater.cc:345] Connected to a master server at 127.19.203.254:46111
02:59:34.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.953032 20865 heartbeater.cc:462] Registering TS with master...
02:59:34.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.954493 20865 heartbeater.cc:508] Master 127.19.203.254:46111 requested a full tablet report, sending...
02:59:34.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.955667 20271 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskKOuND7/build/asan/bin/kudu as pid 20728
02:59:34.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.955890 20271 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-1/wal/instance
02:59:34.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.956951 20313 ts_manager.cc:210] Registered new tserver with Master: decd50576be846f1a20a1684c18ab77f (127.19.203.194:41089)
02:59:34.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.960609 20864 heartbeater.cc:345] Connected to a master server at 127.19.203.252:40563
02:59:34.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.961030 20864 heartbeater.cc:462] Registering TS with master...
02:59:34.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.961772 20864 heartbeater.cc:508] Master 127.19.203.252:40563 requested a full tablet report, sending...
02:59:34.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.962957 20475 ts_manager.cc:210] Registered new tserver with Master: decd50576be846f1a20a1684c18ab77f (127.19.203.194:41089)
02:59:34.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.964425 20475 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.19.203.194:35199
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.964869 20271 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskKOuND7/build/asan/bin/kudu
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKOuND7/build/asan/bin/kudu
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-2/wal
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-2/data
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-2/logs
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-2/data/info.pb
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.203.195:0
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.203.195
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.203.195
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.203.254:46111,127.19.203.253:34409,127.19.203.252:40563
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.203.212:34207
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:59:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
02:59:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
02:59:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
02:59:34.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.970898 20866 heartbeater.cc:345] Connected to a master server at 127.19.203.253:34409
02:59:34.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.971065 20866 heartbeater.cc:462] Registering TS with master...
02:59:34.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.971451 20866 heartbeater.cc:508] Master 127.19.203.253:34409 requested a full tablet report, sending...
02:59:34.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:34.972638 20393 ts_manager.cc:210] Registered new tserver with Master: decd50576be846f1a20a1684c18ab77f (127.19.203.194:41089)
02:59:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:35.321640 20869 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:59:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:35.321920 20869 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:59:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:35.322022 20869 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:59:35.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:35.330780 20869 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:59:35.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:35.330926 20869 flags.cc:432] Enabled experimental flag: --txn_keepalive_interval_ms=1000
02:59:35.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:35.330991 20869 flags.cc:432] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
02:59:35.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:35.331028 20869 flags.cc:432] Enabled experimental flag: --enable_txn_system_client_init=true
02:59:35.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:35.331112 20869 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.203.195
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.341130 20869 tablet_server_runner.cc:78] Tablet server non-default flags:
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-2/logs
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.203.212:34207
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-2/data
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-2/wal
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.203.195:0
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-2/data/info.pb
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.203.195
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.203.254:46111,127.19.203.253:34409,127.19.203.252:40563
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:59:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.203.195
02:59:35.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:59:35.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:59:35.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:59:35.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 05464f8854a90597ddf8dfd45c2061c63b0b716b
02:59:35.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:59:35.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 21 Nov 2026 02:43:19 UTC on bdcb31816ec0
02:59:35.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14886
02:59:35.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
02:59:35.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.343040 20869 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:59:35.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.344988 20869 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:59:35.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:35.355970 20877 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:59:35.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:35.356010 20874 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:59:35.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:35.356144 20875 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:59:35.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.357088 20869 server_base.cc:1061] running on GCE node
02:59:35.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.357702 20869 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:59:35.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.358691 20869 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:59:35.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.359989 20869 hybrid_clock.cc:648] HybridClock initialized: now 1795229975359876 us; error 49 us; skew 500 ppm
02:59:35.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.360396 20869 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:59:35.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.362715 20869 webserver.cc:577] Webserver started at http://127.19.203.195:46725/ using document root <none> and password file <none>
02:59:35.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.363442 20869 fs_manager.cc:362] Metadata directory not provided
02:59:35.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.363585 20869 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:59:35.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.363970 20869 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:59:35.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.366557 20869 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-2/data/instance:
02:59:35.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ccc674be75f448bb2a2da3c962d94c7"
02:59:35.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-21 02:59:35 on dist-test-slave-mbqs"
02:59:35.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.367470 20869 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-2/wal/instance:
02:59:35.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ccc674be75f448bb2a2da3c962d94c7"
02:59:35.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-21 02:59:35 on dist-test-slave-mbqs"
02:59:35.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.373577 20869 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.003s
02:59:35.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.377386 20883 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:59:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.378892 20869 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
02:59:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.379091 20869 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-2/data,/tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-2/wal
02:59:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ccc674be75f448bb2a2da3c962d94c7"
02:59:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-21 02:59:35 on dist-test-slave-mbqs"
02:59:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.379341 20869 fs_report.cc:389] FS layout report
02:59:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:59:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-2/wal
02:59:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-2/wal
02:59:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-2/data/data
02:59:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:59:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:59:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:59:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:59:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:59:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:59:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:59:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:59:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:59:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:59:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:59:35.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.404714 20869 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:59:35.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.405810 20869 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:59:35.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.406159 20869 kserver.cc:163] Server-wide thread pool size limit: 3276
02:59:35.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.409521 20869 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:59:35.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.409652 20869 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:59:35.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.409761 20869 ts_tablet_manager.cc:662] Registered 0 tablets
02:59:35.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.409847 20869 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:59:35.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.448274 20723 heartbeater.cc:500] Master 127.19.203.252:40563 was elected leader, sending a full tablet report...
02:59:35.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.475859 20869 rpc_server.cc:307] RPC server started. Bound to: 127.19.203.195:46645
02:59:35.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.475888 21004 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.203.195:46645 every 8 connection(s)
02:59:35.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.477602 20869 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-2/data/info.pb
02:59:35.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.481611 21006 heartbeater.cc:345] Connected to a master server at 127.19.203.254:46111
02:59:35.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.482179 21006 heartbeater.cc:462] Registering TS with master...
02:59:35.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.483546 21006 heartbeater.cc:508] Master 127.19.203.254:46111 requested a full tablet report, sending...
02:59:35.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.485723 20313 ts_manager.cc:210] Registered new tserver with Master: 2ccc674be75f448bb2a2da3c962d94c7 (127.19.203.195:46645)
02:59:35.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.487162 20271 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskKOuND7/build/asan/bin/kudu as pid 20869
02:59:35.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.487375 20271 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskKOuND7/test-tmp/mini-kudu-cluster18331294424443869606/ts-2/wal/instance
02:59:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.491415 21005 heartbeater.cc:345] Connected to a master server at 127.19.203.252:40563
02:59:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.491590 21005 heartbeater.cc:462] Registering TS with master...
02:59:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.492164 21005 heartbeater.cc:508] Master 127.19.203.252:40563 requested a full tablet report, sending...
02:59:35.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.493306 20475 ts_manager.cc:210] Registered new tserver with Master: 2ccc674be75f448bb2a2da3c962d94c7 (127.19.203.195:46645)
02:59:35.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.494333 20475 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.19.203.195:40881
02:59:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.498675 21007 heartbeater.cc:345] Connected to a master server at 127.19.203.253:34409
02:59:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.498894 21007 heartbeater.cc:462] Registering TS with master...
02:59:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.499348 21007 heartbeater.cc:508] Master 127.19.203.253:34409 requested a full tablet report, sending...
02:59:35.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.500458 20393 ts_manager.cc:210] Registered new tserver with Master: 2ccc674be75f448bb2a2da3c962d94c7 (127.19.203.195:46645)
02:59:35.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.511981 20271 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
02:59:35.514 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:59:35.514 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
02:59:35.515 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.19.203.254"
port: 46111
}
bound_http_address {
host: "127.19.203.254"
port: 34241
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.19.203.253"
port: 34409
}
bound_http_address {
host: "127.19.203.253"
port: 35309
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.19.203.252"
port: 40563
}
bound_http_address {
host: "127.19.203.252"
port: 35205
}
}
}
02:59:35.515 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:59:35.516 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.19.203.193"
port: 37543
}
bound_http_address {
host: "127.19.203.193"
port: 46849
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.19.203.194"
port: 41089
}
bound_http_address {
host: "127.19.203.194"
port: 32833
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.19.203.195"
port: 46645
}
bound_http_address {
host: "127.19.203.195"
port: 46725
}
}
}
02:59:35.516 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
02:59:35.518 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
02:59:35.519 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.203.253:34409(127.19.203.253:34409)] connecting to peer
02:59:35.521 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:35.521 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.19.203.253:34409(127.19.203.253:34409)] Successfully connected to peer
02:59:35.523 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:35.523 [DEBUG - kudu-nio-0] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.19.203.253
02:59:35.525 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.19.203.254:46111(127.19.203.254:46111)] Successfully connected to peer
02:59:35.527 [DEBUG - kudu-nio-1] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.19.203.254
02:59:35.529 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.19.203.252:40563(127.19.203.252:40563)] Successfully connected to peer
02:59:35.531 [DEBUG - kudu-nio-2] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.19.203.252
02:59:35.539 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0xbd223312, L:/127.0.0.1:51628 - R:/127.19.203.253:34409] using SASL/PLAIN
02:59:35.540 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x126116f5, L:/127.0.0.1:39212 - R:/127.19.203.254:46111] using SASL/PLAIN
02:59:35.542 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0x34f10abd, L:/127.0.0.1:36414 - R:/127.19.203.252:40563] using SASL/PLAIN
02:59:35.544 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
02:59:35.544 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=25), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [25ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: OK, deferred=null) after lookup
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.546141 20475 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:36414:
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
02:59:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:35.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:35.548429 20475 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
02:59:35.595 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table fad88b9f6aa644e8a17e7236195459be
02:59:35.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.621490 20940 tablet_service.cc:1495] Processing CreateTablet for tablet 78a62c4f133447e680950d0bc163f1c3 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=fad88b9f6aa644e8a17e7236195459be]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
02:59:35.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.623433 20940 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 78a62c4f133447e680950d0bc163f1c3. 1 dirs total, 0 dirs full, 0 dirs failed
02:59:35.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.621547 20939 tablet_service.cc:1495] Processing CreateTablet for tablet 075c9def146a41cb90f201a35e04d155 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=fad88b9f6aa644e8a17e7236195459be]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
02:59:35.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.627324 20939 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 075c9def146a41cb90f201a35e04d155. 1 dirs total, 0 dirs full, 0 dirs failed
02:59:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.633888 20798 tablet_service.cc:1495] Processing CreateTablet for tablet 075c9def146a41cb90f201a35e04d155 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=fad88b9f6aa644e8a17e7236195459be]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
02:59:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.635478 20798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 075c9def146a41cb90f201a35e04d155. 1 dirs total, 0 dirs full, 0 dirs failed
02:59:35.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.636795 20799 tablet_service.cc:1495] Processing CreateTablet for tablet 78a62c4f133447e680950d0bc163f1c3 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=fad88b9f6aa644e8a17e7236195459be]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
02:59:35.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.637734 20799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 78a62c4f133447e680950d0bc163f1c3. 1 dirs total, 0 dirs full, 0 dirs failed
02:59:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.642136 20658 tablet_service.cc:1495] Processing CreateTablet for tablet 78a62c4f133447e680950d0bc163f1c3 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=fad88b9f6aa644e8a17e7236195459be]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
02:59:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.643970 20658 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 78a62c4f133447e680950d0bc163f1c3. 1 dirs total, 0 dirs full, 0 dirs failed
02:59:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.644088 20657 tablet_service.cc:1495] Processing CreateTablet for tablet 075c9def146a41cb90f201a35e04d155 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=fad88b9f6aa644e8a17e7236195459be]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
02:59:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.644869 20657 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 075c9def146a41cb90f201a35e04d155. 1 dirs total, 0 dirs full, 0 dirs failed
02:59:35.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.653702 21032 tablet_bootstrap.cc:492] T 78a62c4f133447e680950d0bc163f1c3 P 2ccc674be75f448bb2a2da3c962d94c7: Bootstrap starting.
02:59:35.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.656782 21033 tablet_bootstrap.cc:492] T 78a62c4f133447e680950d0bc163f1c3 P decd50576be846f1a20a1684c18ab77f: Bootstrap starting.
02:59:35.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.658412 21032 tablet_bootstrap.cc:654] T 78a62c4f133447e680950d0bc163f1c3 P 2ccc674be75f448bb2a2da3c962d94c7: Neither blocks nor log segments found. Creating new log.
02:59:35.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.659914 21032 log.cc:826] T 78a62c4f133447e680950d0bc163f1c3 P 2ccc674be75f448bb2a2da3c962d94c7: Log is configured to *not* fsync() on all Append() calls
02:59:35.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.663654 21033 tablet_bootstrap.cc:654] T 78a62c4f133447e680950d0bc163f1c3 P decd50576be846f1a20a1684c18ab77f: Neither blocks nor log segments found. Creating new log.
02:59:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.665271 21033 log.cc:826] T 78a62c4f133447e680950d0bc163f1c3 P decd50576be846f1a20a1684c18ab77f: Log is configured to *not* fsync() on all Append() calls
02:59:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.665321 21034 tablet_bootstrap.cc:492] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c: Bootstrap starting.
02:59:35.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.668143 21032 tablet_bootstrap.cc:492] T 78a62c4f133447e680950d0bc163f1c3 P 2ccc674be75f448bb2a2da3c962d94c7: No bootstrap required, opened a new log
02:59:35.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.668376 21033 tablet_bootstrap.cc:492] T 78a62c4f133447e680950d0bc163f1c3 P decd50576be846f1a20a1684c18ab77f: No bootstrap required, opened a new log
02:59:35.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.668481 21032 ts_tablet_manager.cc:1449] T 78a62c4f133447e680950d0bc163f1c3 P 2ccc674be75f448bb2a2da3c962d94c7: Time spent bootstrapping tablet: real 0.015s user 0.002s sys 0.010s
02:59:35.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.668655 21033 ts_tablet_manager.cc:1449] T 78a62c4f133447e680950d0bc163f1c3 P decd50576be846f1a20a1684c18ab77f: Time spent bootstrapping tablet: real 0.015s user 0.009s sys 0.000s
02:59:35.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.669313 21034 tablet_bootstrap.cc:654] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c: Neither blocks nor log segments found. Creating new log.
02:59:35.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.670562 21034 log.cc:826] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c: Log is configured to *not* fsync() on all Append() calls
02:59:35.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.673261 21034 tablet_bootstrap.cc:492] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c: No bootstrap required, opened a new log
02:59:35.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.673542 21034 ts_tablet_manager.cc:1449] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c: Time spent bootstrapping tablet: real 0.009s user 0.008s sys 0.000s
02:59:35.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.674095 21032 raft_consensus.cc:366] T 78a62c4f133447e680950d0bc163f1c3 P 2ccc674be75f448bb2a2da3c962d94c7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } }
02:59:35.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.674453 21032 raft_consensus.cc:392] T 78a62c4f133447e680950d0bc163f1c3 P 2ccc674be75f448bb2a2da3c962d94c7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:59:35.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.674602 21032 raft_consensus.cc:787] T 78a62c4f133447e680950d0bc163f1c3 P 2ccc674be75f448bb2a2da3c962d94c7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ccc674be75f448bb2a2da3c962d94c7, State: Initialized, Role: FOLLOWER
02:59:35.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.675366 21032 consensus_queue.cc:261] T 78a62c4f133447e680950d0bc163f1c3 P 2ccc674be75f448bb2a2da3c962d94c7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } }
02:59:35.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.675848 21033 raft_consensus.cc:366] T 78a62c4f133447e680950d0bc163f1c3 P decd50576be846f1a20a1684c18ab77f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } }
02:59:35.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.676434 21005 heartbeater.cc:500] Master 127.19.203.252:40563 was elected leader, sending a full tablet report...
02:59:35.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.676673 21033 raft_consensus.cc:392] T 78a62c4f133447e680950d0bc163f1c3 P decd50576be846f1a20a1684c18ab77f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:59:35.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.676889 21033 raft_consensus.cc:787] T 78a62c4f133447e680950d0bc163f1c3 P decd50576be846f1a20a1684c18ab77f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: decd50576be846f1a20a1684c18ab77f, State: Initialized, Role: FOLLOWER
02:59:35.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.677651 21033 consensus_queue.cc:261] T 78a62c4f133447e680950d0bc163f1c3 P decd50576be846f1a20a1684c18ab77f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } }
02:59:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.679402 21032 ts_tablet_manager.cc:1480] T 78a62c4f133447e680950d0bc163f1c3 P 2ccc674be75f448bb2a2da3c962d94c7: Time spent starting tablet: real 0.011s user 0.007s sys 0.004s
02:59:35.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.679848 21032 tablet_bootstrap.cc:492] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7: Bootstrap starting.
02:59:35.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.681874 21034 raft_consensus.cc:366] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } }
02:59:35.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.682246 21034 raft_consensus.cc:392] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:59:35.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.682343 21034 raft_consensus.cc:787] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1224c25229164dc7bda79147397a905c, State: Initialized, Role: FOLLOWER
02:59:35.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.682991 21034 consensus_queue.cc:261] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } }
02:59:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.685303 21032 tablet_bootstrap.cc:654] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7: Neither blocks nor log segments found. Creating new log.
02:59:35.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.687335 21034 ts_tablet_manager.cc:1480] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c: Time spent starting tablet: real 0.014s user 0.009s sys 0.003s
02:59:35.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.687916 21033 ts_tablet_manager.cc:1480] T 78a62c4f133447e680950d0bc163f1c3 P decd50576be846f1a20a1684c18ab77f: Time spent starting tablet: real 0.019s user 0.007s sys 0.004s
02:59:35.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.689069 20864 heartbeater.cc:500] Master 127.19.203.252:40563 was elected leader, sending a full tablet report...
02:59:35.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.691480 21032 tablet_bootstrap.cc:492] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7: No bootstrap required, opened a new log
02:59:35.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.691563 21033 tablet_bootstrap.cc:492] T 075c9def146a41cb90f201a35e04d155 P decd50576be846f1a20a1684c18ab77f: Bootstrap starting.
02:59:35.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.691639 21032 ts_tablet_manager.cc:1449] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7: Time spent bootstrapping tablet: real 0.012s user 0.005s sys 0.001s
02:59:35.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.692530 21034 tablet_bootstrap.cc:492] T 075c9def146a41cb90f201a35e04d155 P 1224c25229164dc7bda79147397a905c: Bootstrap starting.
02:59:35.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.692468 21032 raft_consensus.cc:366] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } }
02:59:35.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.692652 21032 raft_consensus.cc:392] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:59:35.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.692736 21032 raft_consensus.cc:787] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ccc674be75f448bb2a2da3c962d94c7, State: Initialized, Role: FOLLOWER
02:59:35.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.693048 21032 consensus_queue.cc:261] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } }
02:59:35.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.695192 21034 tablet_bootstrap.cc:654] T 075c9def146a41cb90f201a35e04d155 P 1224c25229164dc7bda79147397a905c: Neither blocks nor log segments found. Creating new log.
02:59:35.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.695286 21032 ts_tablet_manager.cc:1480] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
02:59:35.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.696734 21033 tablet_bootstrap.cc:654] T 075c9def146a41cb90f201a35e04d155 P decd50576be846f1a20a1684c18ab77f: Neither blocks nor log segments found. Creating new log.
02:59:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.700029 21033 tablet_bootstrap.cc:492] T 075c9def146a41cb90f201a35e04d155 P decd50576be846f1a20a1684c18ab77f: No bootstrap required, opened a new log
02:59:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.700311 21033 ts_tablet_manager.cc:1449] T 075c9def146a41cb90f201a35e04d155 P decd50576be846f1a20a1684c18ab77f: Time spent bootstrapping tablet: real 0.009s user 0.001s sys 0.003s
02:59:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:35.700691 20727 tablet.cc:2554] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:59:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.701320 21033 raft_consensus.cc:366] T 075c9def146a41cb90f201a35e04d155 P decd50576be846f1a20a1684c18ab77f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } }
02:59:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.701517 21033 raft_consensus.cc:392] T 075c9def146a41cb90f201a35e04d155 P decd50576be846f1a20a1684c18ab77f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:59:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.701603 21033 raft_consensus.cc:787] T 075c9def146a41cb90f201a35e04d155 P decd50576be846f1a20a1684c18ab77f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: decd50576be846f1a20a1684c18ab77f, State: Initialized, Role: FOLLOWER
02:59:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.701823 21033 consensus_queue.cc:261] T 075c9def146a41cb90f201a35e04d155 P decd50576be846f1a20a1684c18ab77f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } }
02:59:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.702396 21033 ts_tablet_manager.cc:1480] T 075c9def146a41cb90f201a35e04d155 P decd50576be846f1a20a1684c18ab77f: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
02:59:35.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.703902 21034 tablet_bootstrap.cc:492] T 075c9def146a41cb90f201a35e04d155 P 1224c25229164dc7bda79147397a905c: No bootstrap required, opened a new log
02:59:35.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.704070 21034 ts_tablet_manager.cc:1449] T 075c9def146a41cb90f201a35e04d155 P 1224c25229164dc7bda79147397a905c: Time spent bootstrapping tablet: real 0.012s user 0.002s sys 0.003s
02:59:35.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.704967 21034 raft_consensus.cc:366] T 075c9def146a41cb90f201a35e04d155 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } }
02:59:35.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.705209 21034 raft_consensus.cc:392] T 075c9def146a41cb90f201a35e04d155 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:59:35.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.705305 21034 raft_consensus.cc:787] T 075c9def146a41cb90f201a35e04d155 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1224c25229164dc7bda79147397a905c, State: Initialized, Role: FOLLOWER
02:59:35.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.705592 21034 consensus_queue.cc:261] T 075c9def146a41cb90f201a35e04d155 P 1224c25229164dc7bda79147397a905c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } }
02:59:35.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.706461 21034 ts_tablet_manager.cc:1480] T 075c9def146a41cb90f201a35e04d155 P 1224c25229164dc7bda79147397a905c: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
02:59:35.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:35.712932 20867 tablet.cc:2554] T 075c9def146a41cb90f201a35e04d155 P decd50576be846f1a20a1684c18ab77f: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:59:35.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:35.736553 21008 tablet.cc:2554] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:59:35.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.835822 21040 raft_consensus.cc:534] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:59:35.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.836086 21040 raft_consensus.cc:556] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } }
02:59:35.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.838398 21040 leader_election.cc:290] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2ccc674be75f448bb2a2da3c962d94c7 (127.19.203.195:46645), decd50576be846f1a20a1684c18ab77f (127.19.203.194:41089)
02:59:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.846779 20819 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "78a62c4f133447e680950d0bc163f1c3" candidate_uuid: "1224c25229164dc7bda79147397a905c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "decd50576be846f1a20a1684c18ab77f" is_pre_election: true
02:59:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.847368 20819 raft_consensus.cc:2062] T 78a62c4f133447e680950d0bc163f1c3 P decd50576be846f1a20a1684c18ab77f [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 1224c25229164dc7bda79147397a905c in term 0
02:59:35.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.848148 20602 leader_election.cc:304] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1224c25229164dc7bda79147397a905c, decd50576be846f1a20a1684c18ab77f; no voters:
02:59:35.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.848474 20960 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "78a62c4f133447e680950d0bc163f1c3" candidate_uuid: "1224c25229164dc7bda79147397a905c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2ccc674be75f448bb2a2da3c962d94c7" is_pre_election: true
02:59:35.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.848594 21040 raft_consensus.cc:3024] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Leader pre-election won for term 1
02:59:35.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.848806 21040 raft_consensus.cc:534] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:59:35.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.848930 21040 raft_consensus.cc:3296] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Advancing to term 1
02:59:35.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.849120 20960 raft_consensus.cc:2062] T 78a62c4f133447e680950d0bc163f1c3 P 2ccc674be75f448bb2a2da3c962d94c7 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 1224c25229164dc7bda79147397a905c in term 0
02:59:35.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.849944 21040 raft_consensus.cc:556] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } }
02:59:35.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.852944 21040 leader_election.cc:290] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c [CANDIDATE]: Term 1 election: Requested vote from peers 2ccc674be75f448bb2a2da3c962d94c7 (127.19.203.195:46645), decd50576be846f1a20a1684c18ab77f (127.19.203.194:41089)
02:59:35.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.853339 20960 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "78a62c4f133447e680950d0bc163f1c3" candidate_uuid: "1224c25229164dc7bda79147397a905c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2ccc674be75f448bb2a2da3c962d94c7"
02:59:35.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.853585 20960 raft_consensus.cc:3296] T 78a62c4f133447e680950d0bc163f1c3 P 2ccc674be75f448bb2a2da3c962d94c7 [term 0 FOLLOWER]: Advancing to term 1
02:59:35.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.853804 20819 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "78a62c4f133447e680950d0bc163f1c3" candidate_uuid: "1224c25229164dc7bda79147397a905c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "decd50576be846f1a20a1684c18ab77f"
02:59:35.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.854075 20960 raft_consensus.cc:2062] T 78a62c4f133447e680950d0bc163f1c3 P 2ccc674be75f448bb2a2da3c962d94c7 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 1224c25229164dc7bda79147397a905c in term 1
02:59:35.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.854012 20819 raft_consensus.cc:3296] T 78a62c4f133447e680950d0bc163f1c3 P decd50576be846f1a20a1684c18ab77f [term 0 FOLLOWER]: Advancing to term 1
02:59:35.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.854523 20819 raft_consensus.cc:2062] T 78a62c4f133447e680950d0bc163f1c3 P decd50576be846f1a20a1684c18ab77f [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 1224c25229164dc7bda79147397a905c in term 1
02:59:35.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.858119 20602 leader_election.cc:304] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1224c25229164dc7bda79147397a905c, decd50576be846f1a20a1684c18ab77f; no voters:
02:59:35.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.858402 21040 raft_consensus.cc:3024] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c [term 1 FOLLOWER]: Leader election won for term 1
02:59:35.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.858595 21040 raft_consensus.cc:744] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c [term 1 LEADER]: Becoming Leader. State: Replica: 1224c25229164dc7bda79147397a905c, State: Running, Role: LEADER
02:59:35.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.859196 21040 consensus_queue.cc:238] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } }
02:59:35.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.866681 21046 consensus_queue.cc:1049] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c [LEADER]: Connected to new peer: Peer: permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:59:35.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.867821 20474 catalog_manager.cc:5856] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c reported cstate change: term changed from 0 to 1, leader changed from <none> to 1224c25229164dc7bda79147397a905c (127.19.203.193). New cstate: current_term: 1 leader_uuid: "1224c25229164dc7bda79147397a905c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } health_report { overall_health: HEALTHY } } }
02:59:35.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.871297 21046 consensus_queue.cc:1049] T 78a62c4f133447e680950d0bc163f1c3 P 1224c25229164dc7bda79147397a905c [LEADER]: Connected to new peer: Peer: permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:59:35.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.964032 21038 raft_consensus.cc:534] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:59:35.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.964254 21038 raft_consensus.cc:556] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } }
02:59:35.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.966001 21038 leader_election.cc:290] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1224c25229164dc7bda79147397a905c (127.19.203.193:37543), decd50576be846f1a20a1684c18ab77f (127.19.203.194:41089)
02:59:35.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.973449 20819 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "075c9def146a41cb90f201a35e04d155" candidate_uuid: "2ccc674be75f448bb2a2da3c962d94c7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "decd50576be846f1a20a1684c18ab77f" is_pre_election: true
02:59:35.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.973737 20678 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "075c9def146a41cb90f201a35e04d155" candidate_uuid: "2ccc674be75f448bb2a2da3c962d94c7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1224c25229164dc7bda79147397a905c" is_pre_election: true
02:59:35.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.973891 20819 raft_consensus.cc:2062] T 075c9def146a41cb90f201a35e04d155 P decd50576be846f1a20a1684c18ab77f [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 2ccc674be75f448bb2a2da3c962d94c7 in term 0
02:59:35.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.974220 20678 raft_consensus.cc:2062] T 075c9def146a41cb90f201a35e04d155 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 2ccc674be75f448bb2a2da3c962d94c7 in term 0
02:59:35.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.974419 20887 leader_election.cc:304] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ccc674be75f448bb2a2da3c962d94c7, decd50576be846f1a20a1684c18ab77f; no voters:
02:59:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.974876 21038 raft_consensus.cc:3024] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:59:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.975019 21038 raft_consensus.cc:534] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:59:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.975131 21038 raft_consensus.cc:3296] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7 [term 0 FOLLOWER]: Advancing to term 1
02:59:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.975678 21038 raft_consensus.cc:556] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } }
02:59:35.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.978282 21038 leader_election.cc:290] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7 [CANDIDATE]: Term 1 election: Requested vote from peers 1224c25229164dc7bda79147397a905c (127.19.203.193:37543), decd50576be846f1a20a1684c18ab77f (127.19.203.194:41089)
02:59:35.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.978796 20678 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "075c9def146a41cb90f201a35e04d155" candidate_uuid: "2ccc674be75f448bb2a2da3c962d94c7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1224c25229164dc7bda79147397a905c"
02:59:35.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.978958 20678 raft_consensus.cc:3296] T 075c9def146a41cb90f201a35e04d155 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Advancing to term 1
02:59:35.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.978848 20819 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "075c9def146a41cb90f201a35e04d155" candidate_uuid: "2ccc674be75f448bb2a2da3c962d94c7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "decd50576be846f1a20a1684c18ab77f"
02:59:35.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.979148 20819 raft_consensus.cc:3296] T 075c9def146a41cb90f201a35e04d155 P decd50576be846f1a20a1684c18ab77f [term 0 FOLLOWER]: Advancing to term 1
02:59:35.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.979298 20678 raft_consensus.cc:2062] T 075c9def146a41cb90f201a35e04d155 P 1224c25229164dc7bda79147397a905c [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 2ccc674be75f448bb2a2da3c962d94c7 in term 1
02:59:35.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.979539 20819 raft_consensus.cc:2062] T 075c9def146a41cb90f201a35e04d155 P decd50576be846f1a20a1684c18ab77f [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 2ccc674be75f448bb2a2da3c962d94c7 in term 1
02:59:35.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.982707 20887 leader_election.cc:304] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ccc674be75f448bb2a2da3c962d94c7, decd50576be846f1a20a1684c18ab77f; no voters:
02:59:35.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.983067 21038 raft_consensus.cc:3024] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7 [term 1 FOLLOWER]: Leader election won for term 1
02:59:35.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.983237 21038 raft_consensus.cc:744] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7 [term 1 LEADER]: Becoming Leader. State: Replica: 2ccc674be75f448bb2a2da3c962d94c7, State: Running, Role: LEADER
02:59:35.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.983700 21038 consensus_queue.cc:238] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } }
02:59:35.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.992252 21055 consensus_queue.cc:1049] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:59:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.993767 20474 catalog_manager.cc:5856] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7 reported cstate change: term changed from 0 to 1, leader changed from <none> to 2ccc674be75f448bb2a2da3c962d94c7 (127.19.203.195). New cstate: current_term: 1 leader_uuid: "2ccc674be75f448bb2a2da3c962d94c7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } health_report { overall_health: HEALTHY } } }
02:59:36.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:35.997263 21057 consensus_queue.cc:1049] T 075c9def146a41cb90f201a35e04d155 P 2ccc674be75f448bb2a2da3c962d94c7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:59:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.907270 20475 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:39388:
02:59:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
02:59:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
02:59:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:59:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
02:59:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
02:59:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
02:59:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:59:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
02:59:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
02:59:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:59:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:59:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:59:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:59:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
02:59:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
02:59:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
02:59:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
02:59:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
02:59:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
02:59:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:59:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
02:59:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
02:59:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:36.908546 20475 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
02:59:36.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.922645 20658 tablet_service.cc:1495] Processing CreateTablet for tablet 060f7f4739054f97bdf1e661323e0466 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=a64f12279f7d48c9b0cac6288c473ede]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
02:59:36.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.923156 20798 tablet_service.cc:1495] Processing CreateTablet for tablet 060f7f4739054f97bdf1e661323e0466 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=a64f12279f7d48c9b0cac6288c473ede]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
02:59:36.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.923782 20658 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 060f7f4739054f97bdf1e661323e0466. 1 dirs total, 0 dirs full, 0 dirs failed
02:59:36.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.923983 20798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 060f7f4739054f97bdf1e661323e0466. 1 dirs total, 0 dirs full, 0 dirs failed
02:59:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.927392 20940 tablet_service.cc:1495] Processing CreateTablet for tablet 060f7f4739054f97bdf1e661323e0466 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=a64f12279f7d48c9b0cac6288c473ede]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
02:59:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.928328 20940 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 060f7f4739054f97bdf1e661323e0466. 1 dirs total, 0 dirs full, 0 dirs failed
02:59:36.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.932399 21080 tablet_bootstrap.cc:492] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c: Bootstrap starting.
02:59:36.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.934744 21080 tablet_bootstrap.cc:654] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c: Neither blocks nor log segments found. Creating new log.
02:59:36.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.935339 21082 tablet_bootstrap.cc:492] T 060f7f4739054f97bdf1e661323e0466 P 2ccc674be75f448bb2a2da3c962d94c7: Bootstrap starting.
02:59:36.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.936838 21080 tablet_bootstrap.cc:492] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c: No bootstrap required, opened a new log
02:59:36.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.936988 21080 ts_tablet_manager.cc:1449] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.001s
02:59:36.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.937017 21081 tablet_bootstrap.cc:492] T 060f7f4739054f97bdf1e661323e0466 P decd50576be846f1a20a1684c18ab77f: Bootstrap starting.
02:59:36.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.937414 21082 tablet_bootstrap.cc:654] T 060f7f4739054f97bdf1e661323e0466 P 2ccc674be75f448bb2a2da3c962d94c7: Neither blocks nor log segments found. Creating new log.
02:59:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.937754 21080 raft_consensus.cc:366] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } }
02:59:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.937983 21080 raft_consensus.cc:392] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:59:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.938081 21080 raft_consensus.cc:787] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1224c25229164dc7bda79147397a905c, State: Initialized, Role: FOLLOWER
02:59:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.938339 21080 consensus_queue.cc:261] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } }
02:59:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.938896 21081 tablet_bootstrap.cc:654] T 060f7f4739054f97bdf1e661323e0466 P decd50576be846f1a20a1684c18ab77f: Neither blocks nor log segments found. Creating new log.
02:59:36.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.939126 21040 tablet_replica.cc:442] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } } }
02:59:36.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.939373 21040 tablet_replica.cc:445] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c: This TxnStatusTablet replica's current role is: FOLLOWER
02:59:36.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.940608 21082 tablet_bootstrap.cc:492] T 060f7f4739054f97bdf1e661323e0466 P 2ccc674be75f448bb2a2da3c962d94c7: No bootstrap required, opened a new log
02:59:36.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.940747 21082 ts_tablet_manager.cc:1449] T 060f7f4739054f97bdf1e661323e0466 P 2ccc674be75f448bb2a2da3c962d94c7: Time spent bootstrapping tablet: real 0.006s user 0.004s sys 0.000s
02:59:36.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.941600 21082 raft_consensus.cc:366] T 060f7f4739054f97bdf1e661323e0466 P 2ccc674be75f448bb2a2da3c962d94c7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } }
02:59:36.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.941776 21082 raft_consensus.cc:392] T 060f7f4739054f97bdf1e661323e0466 P 2ccc674be75f448bb2a2da3c962d94c7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:59:36.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.941874 21082 raft_consensus.cc:787] T 060f7f4739054f97bdf1e661323e0466 P 2ccc674be75f448bb2a2da3c962d94c7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ccc674be75f448bb2a2da3c962d94c7, State: Initialized, Role: FOLLOWER
02:59:36.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.942091 21082 consensus_queue.cc:261] T 060f7f4739054f97bdf1e661323e0466 P 2ccc674be75f448bb2a2da3c962d94c7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } }
02:59:36.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.942495 21060 tablet_replica.cc:442] T 060f7f4739054f97bdf1e661323e0466 P 2ccc674be75f448bb2a2da3c962d94c7: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } } }
02:59:36.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.942638 21082 ts_tablet_manager.cc:1480] T 060f7f4739054f97bdf1e661323e0466 P 2ccc674be75f448bb2a2da3c962d94c7: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
02:59:36.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.942719 21060 tablet_replica.cc:445] T 060f7f4739054f97bdf1e661323e0466 P 2ccc674be75f448bb2a2da3c962d94c7: This TxnStatusTablet replica's current role is: FOLLOWER
02:59:36.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.944092 21081 tablet_bootstrap.cc:492] T 060f7f4739054f97bdf1e661323e0466 P decd50576be846f1a20a1684c18ab77f: No bootstrap required, opened a new log
02:59:36.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.944259 21080 ts_tablet_manager.cc:1480] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c: Time spent starting tablet: real 0.007s user 0.000s sys 0.002s
02:59:36.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.944353 21081 ts_tablet_manager.cc:1449] T 060f7f4739054f97bdf1e661323e0466 P decd50576be846f1a20a1684c18ab77f: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.005s
02:59:36.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.945365 21081 raft_consensus.cc:366] T 060f7f4739054f97bdf1e661323e0466 P decd50576be846f1a20a1684c18ab77f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } }
02:59:36.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.945673 21081 raft_consensus.cc:392] T 060f7f4739054f97bdf1e661323e0466 P decd50576be846f1a20a1684c18ab77f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:59:36.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.945863 21081 raft_consensus.cc:787] T 060f7f4739054f97bdf1e661323e0466 P decd50576be846f1a20a1684c18ab77f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: decd50576be846f1a20a1684c18ab77f, State: Initialized, Role: FOLLOWER
02:59:36.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.946610 21081 consensus_queue.cc:261] T 060f7f4739054f97bdf1e661323e0466 P decd50576be846f1a20a1684c18ab77f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } }
02:59:36.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.948741 21081 ts_tablet_manager.cc:1480] T 060f7f4739054f97bdf1e661323e0466 P decd50576be846f1a20a1684c18ab77f: Time spent starting tablet: real 0.004s user 0.003s sys 0.000s
02:59:36.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.949389 21086 tablet_replica.cc:442] T 060f7f4739054f97bdf1e661323e0466 P decd50576be846f1a20a1684c18ab77f: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } } }
02:59:36.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.949771 21086 tablet_replica.cc:445] T 060f7f4739054f97bdf1e661323e0466 P decd50576be846f1a20a1684c18ab77f: This TxnStatusTablet replica's current role is: FOLLOWER
02:59:36.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.997884 21040 raft_consensus.cc:534] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:59:36.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.998139 21040 raft_consensus.cc:556] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } }
02:59:36.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.998900 21040 leader_election.cc:290] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers decd50576be846f1a20a1684c18ab77f (127.19.203.194:41089), 2ccc674be75f448bb2a2da3c962d94c7 (127.19.203.195:46645)
02:59:36.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.999475 20960 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "060f7f4739054f97bdf1e661323e0466" candidate_uuid: "1224c25229164dc7bda79147397a905c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2ccc674be75f448bb2a2da3c962d94c7" is_pre_election: true
02:59:36.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.999495 20819 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "060f7f4739054f97bdf1e661323e0466" candidate_uuid: "1224c25229164dc7bda79147397a905c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "decd50576be846f1a20a1684c18ab77f" is_pre_election: true
02:59:36.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.999758 20819 raft_consensus.cc:2062] T 060f7f4739054f97bdf1e661323e0466 P decd50576be846f1a20a1684c18ab77f [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 1224c25229164dc7bda79147397a905c in term 0
02:59:36.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:36.999778 20960 raft_consensus.cc:2062] T 060f7f4739054f97bdf1e661323e0466 P 2ccc674be75f448bb2a2da3c962d94c7 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 1224c25229164dc7bda79147397a905c in term 0
02:59:37.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.000259 20602 leader_election.cc:304] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1224c25229164dc7bda79147397a905c, decd50576be846f1a20a1684c18ab77f; no voters:
02:59:37.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.000696 21040 raft_consensus.cc:3024] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Leader pre-election won for term 1
02:59:37.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.000835 21040 raft_consensus.cc:534] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:59:37.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.000947 21040 raft_consensus.cc:3296] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c [term 0 FOLLOWER]: Advancing to term 1
02:59:37.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.001394 21040 raft_consensus.cc:556] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } }
02:59:37.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.003841 21040 leader_election.cc:290] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c [CANDIDATE]: Term 1 election: Requested vote from peers decd50576be846f1a20a1684c18ab77f (127.19.203.194:41089), 2ccc674be75f448bb2a2da3c962d94c7 (127.19.203.195:46645)
02:59:37.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.004354 20960 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "060f7f4739054f97bdf1e661323e0466" candidate_uuid: "1224c25229164dc7bda79147397a905c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2ccc674be75f448bb2a2da3c962d94c7"
02:59:37.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.004410 20819 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "060f7f4739054f97bdf1e661323e0466" candidate_uuid: "1224c25229164dc7bda79147397a905c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "decd50576be846f1a20a1684c18ab77f"
02:59:37.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.004558 20819 raft_consensus.cc:3296] T 060f7f4739054f97bdf1e661323e0466 P decd50576be846f1a20a1684c18ab77f [term 0 FOLLOWER]: Advancing to term 1
02:59:37.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.004576 20960 raft_consensus.cc:3296] T 060f7f4739054f97bdf1e661323e0466 P 2ccc674be75f448bb2a2da3c962d94c7 [term 0 FOLLOWER]: Advancing to term 1
02:59:37.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.004886 20819 raft_consensus.cc:2062] T 060f7f4739054f97bdf1e661323e0466 P decd50576be846f1a20a1684c18ab77f [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 1224c25229164dc7bda79147397a905c in term 1
02:59:37.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.004922 20960 raft_consensus.cc:2062] T 060f7f4739054f97bdf1e661323e0466 P 2ccc674be75f448bb2a2da3c962d94c7 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 1224c25229164dc7bda79147397a905c in term 1
02:59:37.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.008155 20602 leader_election.cc:304] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1224c25229164dc7bda79147397a905c, 2ccc674be75f448bb2a2da3c962d94c7; no voters:
02:59:37.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.008420 21040 raft_consensus.cc:3024] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c [term 1 FOLLOWER]: Leader election won for term 1
02:59:37.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.008649 21040 raft_consensus.cc:744] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c [term 1 LEADER]: Becoming Leader. State: Replica: 1224c25229164dc7bda79147397a905c, State: Running, Role: LEADER
02:59:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.008996 21040 consensus_queue.cc:238] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } }
02:59:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.012694 21087 tablet_replica.cc:442] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c: TxnStatusTablet state changed. Reason: New leader 1224c25229164dc7bda79147397a905c. Latest consensus state: current_term: 1 leader_uuid: "1224c25229164dc7bda79147397a905c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } } }
02:59:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.012923 21087 tablet_replica.cc:445] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c: This TxnStatusTablet replica's current role is: LEADER
02:59:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.014371 20475 catalog_manager.cc:5856] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c reported cstate change: term changed from 0 to 1, leader changed from <none> to 1224c25229164dc7bda79147397a905c (127.19.203.193). New cstate: current_term: 1 leader_uuid: "1224c25229164dc7bda79147397a905c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } health_report { overall_health: UNKNOWN } } }
02:59:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.015228 21089 consensus_queue.cc:1049] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c [LEADER]: Connected to new peer: Peer: permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:59:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.023305 21091 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
02:59:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.023604 21091 txn_status_manager.cc:930] Loading transaction status metadata into memory...
02:59:37.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.025765 21060 tablet_replica.cc:442] T 060f7f4739054f97bdf1e661323e0466 P 2ccc674be75f448bb2a2da3c962d94c7: TxnStatusTablet state changed. Reason: New leader 1224c25229164dc7bda79147397a905c. Latest consensus state: current_term: 1 leader_uuid: "1224c25229164dc7bda79147397a905c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } } }
02:59:37.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.026022 21060 tablet_replica.cc:445] T 060f7f4739054f97bdf1e661323e0466 P 2ccc674be75f448bb2a2da3c962d94c7: This TxnStatusTablet replica's current role is: FOLLOWER
02:59:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.029640 21090 consensus_queue.cc:1049] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c [LEADER]: Connected to new peer: Peer: permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:59:37.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.033164 21087 tablet_replica.cc:442] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1224c25229164dc7bda79147397a905c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } } }
02:59:37.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.033339 21087 tablet_replica.cc:445] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c: This TxnStatusTablet replica's current role is: LEADER
02:59:37.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.046099 21060 tablet_replica.cc:442] T 060f7f4739054f97bdf1e661323e0466 P 2ccc674be75f448bb2a2da3c962d94c7: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1224c25229164dc7bda79147397a905c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } } }
02:59:37.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.046319 21060 tablet_replica.cc:445] T 060f7f4739054f97bdf1e661323e0466 P 2ccc674be75f448bb2a2da3c962d94c7: This TxnStatusTablet replica's current role is: FOLLOWER
02:59:37.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.046248 21086 tablet_replica.cc:442] T 060f7f4739054f97bdf1e661323e0466 P decd50576be846f1a20a1684c18ab77f: TxnStatusTablet state changed. Reason: New leader 1224c25229164dc7bda79147397a905c. Latest consensus state: current_term: 1 leader_uuid: "1224c25229164dc7bda79147397a905c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } } }
02:59:37.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.046448 21086 tablet_replica.cc:445] T 060f7f4739054f97bdf1e661323e0466 P decd50576be846f1a20a1684c18ab77f: This TxnStatusTablet replica's current role is: FOLLOWER
02:59:37.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.049129 21087 tablet_replica.cc:442] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1224c25229164dc7bda79147397a905c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } } }
02:59:37.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.049382 21087 tablet_replica.cc:445] T 060f7f4739054f97bdf1e661323e0466 P 1224c25229164dc7bda79147397a905c: This TxnStatusTablet replica's current role is: LEADER
02:59:37.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.057132 21086 tablet_replica.cc:442] T 060f7f4739054f97bdf1e661323e0466 P decd50576be846f1a20a1684c18ab77f: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1224c25229164dc7bda79147397a905c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1224c25229164dc7bda79147397a905c" member_type: VOTER last_known_addr { host: "127.19.203.193" port: 37543 } } peers { permanent_uuid: "decd50576be846f1a20a1684c18ab77f" member_type: VOTER last_known_addr { host: "127.19.203.194" port: 41089 } } peers { permanent_uuid: "2ccc674be75f448bb2a2da3c962d94c7" member_type: VOTER last_known_addr { host: "127.19.203.195" port: 46645 } } }
02:59:37.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.057337 21086 tablet_replica.cc:445] T 060f7f4739054f97bdf1e661323e0466 P decd50576be846f1a20a1684c18ab77f: This TxnStatusTablet replica's current role is: FOLLOWER
02:59:37.096 [DEBUG - Time-limited test] (KuduTransaction.java:705) starting keepalive heartbeating with period 500 ms (txn ID 0)
02:59:37.099 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table fad88b9f6aa644e8a17e7236195459be
02:59:37.099 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.203.252:40563
02:59:37.100 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:59:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.100486 20271 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskKOuND7/build/asan/bin/kudu pid 20441
02:59:37.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.102084 20515 generic_service.cc:196] Checking for leaks (request via RPC)
02:59:37.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.265328 20271 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskKOuND7/build/asan/bin/kudu with pid 20441
02:59:37.296 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.19.203.252:40563(127.19.203.252:40563)] handling channelInactive
02:59:37.296 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state READY due to: connection closed
02:59:37.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:37.297556 20380 connection.cc:573] server connection from 127.0.0.1:51614 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
02:59:37.298 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:59:37.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.566354 21104 raft_consensus.cc:534] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 1 FOLLOWER]: Starting pre-election (detected failure of leader efb4c69b20bd4f9692d496c33ddbb24e)
02:59:37.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.566583 21104 raft_consensus.cc:556] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:37.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.566676 21105 raft_consensus.cc:534] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader efb4c69b20bd4f9692d496c33ddbb24e)
02:59:37.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.566869 21105 raft_consensus.cc:556] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:37.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.567543 21104 leader_election.cc:290] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
02:59:37.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.568123 20322 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1e3c2fda688743cfb0f78de4c81a953c" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "56099a6022da440a81f20e6fe8903fb0" is_pre_election: true
02:59:37.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.568441 21105 leader_election.cc:290] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 1e3c2fda688743cfb0f78de4c81a953c (127.19.203.253:34409), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
02:59:37.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.568499 20322 raft_consensus.cc:2062] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 1 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 1e3c2fda688743cfb0f78de4c81a953c in term 1
02:59:37.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.569120 20403 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "56099a6022da440a81f20e6fe8903fb0" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "1e3c2fda688743cfb0f78de4c81a953c" is_pre_election: true
02:59:37.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.569567 20403 raft_consensus.cc:2062] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 1 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 56099a6022da440a81f20e6fe8903fb0 in term 1
02:59:37.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.569765 20378 leader_election.cc:304] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c, 56099a6022da440a81f20e6fe8903fb0; no voters:
02:59:37.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.570042 21104 raft_consensus.cc:3024] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 1 FOLLOWER]: Leader pre-election won for term 2
02:59:37.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.570206 21104 raft_consensus.cc:534] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 1 FOLLOWER]: Starting leader election (detected failure of leader efb4c69b20bd4f9692d496c33ddbb24e)
02:59:37.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.570340 21104 raft_consensus.cc:3296] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 1 FOLLOWER]: Advancing to term 2
02:59:37.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.570662 20299 leader_election.cc:304] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c, 56099a6022da440a81f20e6fe8903fb0; no voters:
02:59:37.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:37.570777 20300 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111) [suppressed 8 similar messages]
02:59:37.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.571137 21105 raft_consensus.cc:3024] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 1 FOLLOWER]: Leader pre-election won for term 2
02:59:37.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.571378 21105 raft_consensus.cc:534] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 1 FOLLOWER]: Starting leader election (detected failure of leader efb4c69b20bd4f9692d496c33ddbb24e)
02:59:37.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.571288 21104 raft_consensus.cc:556] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:37.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.571681 21105 raft_consensus.cc:3296] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 1 FOLLOWER]: Advancing to term 2
02:59:37.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:37.572386 20381 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:37.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.572696 21105 raft_consensus.cc:556] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:37.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:37.574927 20300 leader_election.cc:336] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:37.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.575163 21104 leader_election.cc:290] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 2 election: Requested vote from peers 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
02:59:37.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.576061 20322 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1e3c2fda688743cfb0f78de4c81a953c" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "56099a6022da440a81f20e6fe8903fb0"
02:59:37.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.576390 20322 raft_consensus.cc:2628] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 1e3c2fda688743cfb0f78de4c81a953c in current term 2: Already voted for candidate 56099a6022da440a81f20e6fe8903fb0 in this term.
02:59:37.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.576529 21105 leader_election.cc:290] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [CANDIDATE]: Term 2 election: Requested vote from peers 1e3c2fda688743cfb0f78de4c81a953c (127.19.203.253:34409), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
02:59:37.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.576856 20403 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "56099a6022da440a81f20e6fe8903fb0" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "1e3c2fda688743cfb0f78de4c81a953c"
02:59:37.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.577162 20403 raft_consensus.cc:2628] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 56099a6022da440a81f20e6fe8903fb0 in current term 2: Already voted for candidate 1e3c2fda688743cfb0f78de4c81a953c in this term.
02:59:37.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:37.577196 20381 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:37.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.577644 20378 leader_election.cc:304] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c; no voters: 56099a6022da440a81f20e6fe8903fb0, efb4c69b20bd4f9692d496c33ddbb24e
02:59:37.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.577978 21104 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
02:59:37.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:37.578763 20300 leader_election.cc:336] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:37.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.578913 20300 leader_election.cc:304] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 56099a6022da440a81f20e6fe8903fb0; no voters: 1e3c2fda688743cfb0f78de4c81a953c, efb4c69b20bd4f9692d496c33ddbb24e
02:59:37.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.579253 21105 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
02:59:37.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:37.602 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.602 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.602 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location master-127.19.203.252:40563(127.19.203.252:40563) for tablet Kudu Master: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:37.621 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.621 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.622 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.622 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:37.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:37.642 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.642 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.642 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.642 [WARN - kudu-nio-5] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:37.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:37.662 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.662 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.662 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.662 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:37.679 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:37.681 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.682 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.682 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.682 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:37.719 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:37.720 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.721 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.721 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.721 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:37.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:37.761 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.761 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.761 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.761 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:37.762 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] sending RPC to server master-127.19.203.252:40563, [2ms] received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@49872714(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$240/0x00007f393c339e30@69c600ed -> (continuation of Deferred@1395578169 after retry RPC after error@1847423498) -> (continuation of Deferred@735756844 after retry RPC after error@1925344978) -> (continuation of Deferred@1465969326 after retry RPC after error@1390960490) -> (continuation of Deferred@803089593 after retry RPC after error@1317757806) -> (continuation of Deferred@510580107 after retry RPC after error@250464803), errback=org.apache.kudu.client.KuduTransaction$$Lambda$241/0x00007f393c33a058@508884a3 -> (continuation of Deferred@1395578169 after retry RPC after error@1847423498) -> (continuation of Deferred@735756844 after retry RPC after error@1925344978) -> (continuation of Deferred@1465969326 after retry RPC after error@1390960490) -> (continuation of Deferred@803089593 after retry RPC after error@1317757806) -> (continuation of Deferred@510580107 after retry RPC after error@250464803)))
02:59:37.762 [DEBUG - kudu-nio-1] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] sending RPC to server master-127.19.203.252:40563, [2ms] received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@49872714(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$240/0x00007f393c339e30@69c600ed -> (continuation of Deferred@1395578169 after retry RPC after error@1847423498) -> (continuation of Deferred@735756844 after retry RPC after error@1925344978) -> (continuation of Deferred@1465969326 after retry RPC after error@1390960490) -> (continuation of Deferred@803089593 after retry RPC after error@1317757806) -> (continuation of Deferred@510580107 after retry RPC after error@250464803), errback=org.apache.kudu.client.KuduTransaction$$Lambda$241/0x00007f393c33a058@508884a3 -> (continuation of Deferred@1395578169 after retry RPC after error@1847423498) -> (continuation of Deferred@735756844 after retry RPC after error@1925344978) -> (continuation of Deferred@1465969326 after retry RPC after error@1390960490) -> (continuation of Deferred@803089593 after retry RPC after error@1317757806) -> (continuation of Deferred@510580107 after retry RPC after error@250464803)))
02:59:37.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:37.781 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.781 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.781 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:37.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:37.801 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.801 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.801 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.801 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:37.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:37.821 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.821 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.821 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:37.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:37.841 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.841 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.843 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.843 [WARN - kudu-nio-5] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:37.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:37.861 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.861 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.861 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.862 [WARN - pool-40-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:37.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:37.881 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.881 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.882 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.882 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:37.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:37.901 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.901 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.901 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.901 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:37.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:37.942 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.942 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.942 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:37.942 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:37.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.990568 21105 raft_consensus.cc:534] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:59:37.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.990849 21105 raft_consensus.cc:556] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:37.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.991539 21105 leader_election.cc:290] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 1e3c2fda688743cfb0f78de4c81a953c (127.19.203.253:34409), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
02:59:37.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.992143 20403 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "56099a6022da440a81f20e6fe8903fb0" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "1e3c2fda688743cfb0f78de4c81a953c" is_pre_election: true
02:59:37.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.992468 20403 raft_consensus.cc:2062] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 2 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 56099a6022da440a81f20e6fe8903fb0 in term 2
02:59:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.992954 20299 leader_election.cc:304] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c, 56099a6022da440a81f20e6fe8903fb0; no voters:
02:59:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:37.993173 20300 leader_election.cc:336] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.993268 21105 raft_consensus.cc:3024] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 2 FOLLOWER]: Leader pre-election won for term 3
02:59:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.993358 21105 raft_consensus.cc:534] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:59:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.993496 21105 raft_consensus.cc:3296] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 2 FOLLOWER]: Advancing to term 3
02:59:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.993748 21105 raft_consensus.cc:556] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:37.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.996088 21105 leader_election.cc:290] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [CANDIDATE]: Term 3 election: Requested vote from peers 1e3c2fda688743cfb0f78de4c81a953c (127.19.203.253:34409), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
02:59:37.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.996450 20403 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "56099a6022da440a81f20e6fe8903fb0" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "1e3c2fda688743cfb0f78de4c81a953c"
02:59:37.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.996680 20403 raft_consensus.cc:3296] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 2 FOLLOWER]: Advancing to term 3
02:59:37.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.997079 20403 raft_consensus.cc:2062] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Leader election vote request: granting yes vote for candidate 56099a6022da440a81f20e6fe8903fb0 in term 3
02:59:37.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.999330 20299 leader_election.cc:304] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c, 56099a6022da440a81f20e6fe8903fb0; no voters:
02:59:37.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.999590 21105 raft_consensus.cc:3024] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 3 FOLLOWER]: Leader election won for term 3
02:59:37.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:37.999643 20300 leader_election.cc:336] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:37.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:37.999791 21105 raft_consensus.cc:744] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [term 3 LEADER]: Becoming Leader. State: Replica: 56099a6022da440a81f20e6fe8903fb0, State: Running, Role: LEADER
02:59:38.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.000281 21105 consensus_queue.cc:238] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.001763 21111 sys_catalog.cc:455] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 56099a6022da440a81f20e6fe8903fb0. Latest consensus state: current_term: 3 leader_uuid: "56099a6022da440a81f20e6fe8903fb0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } } }
02:59:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.002056 21111 sys_catalog.cc:458] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [sys.catalog]: This master's current role is: LEADER
02:59:38.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.003185 21119 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
02:59:38.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.003401 20403 raft_consensus.cc:1338] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Refusing update from remote peer 56099a6022da440a81f20e6fe8903fb0: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
02:59:38.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.004144 21105 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
02:59:38.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:38.004405 20300 consensus_peers.cc:597] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 -> Peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Couldn't send request to peer efb4c69b20bd4f9692d496c33ddbb24e. Status: Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:59:38.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.005686 21104 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 56099a6022da440a81f20e6fe8903fb0. Latest consensus state: current_term: 3 leader_uuid: "56099a6022da440a81f20e6fe8903fb0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } } }
02:59:38.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.005959 21104 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [sys.catalog]: This master's current role is: FOLLOWER
02:59:38.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.006626 21119 catalog_manager.cc:694] Loaded metadata for table kudu_system.kudu_transactions [id=a64f12279f7d48c9b0cac6288c473ede]
02:59:38.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.007453 21119 catalog_manager.cc:694] Loaded metadata for table txn_manager_fallback_rolling [id=fad88b9f6aa644e8a17e7236195459be]
02:59:38.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.008121 21104 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "56099a6022da440a81f20e6fe8903fb0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } } }
02:59:38.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.008333 21104 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [sys.catalog]: This master's current role is: FOLLOWER
02:59:38.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.009429 21111 sys_catalog.cc:455] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "56099a6022da440a81f20e6fe8903fb0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } } }
02:59:38.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.009629 21111 sys_catalog.cc:458] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 [sys.catalog]: This master's current role is: LEADER
02:59:38.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.010128 21119 tablet_loader.cc:96] loaded metadata for tablet 060f7f4739054f97bdf1e661323e0466 (table kudu_system.kudu_transactions [id=a64f12279f7d48c9b0cac6288c473ede])
02:59:38.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.010548 21119 tablet_loader.cc:96] loaded metadata for tablet 075c9def146a41cb90f201a35e04d155 (table txn_manager_fallback_rolling [id=fad88b9f6aa644e8a17e7236195459be])
02:59:38.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.010850 21119 tablet_loader.cc:96] loaded metadata for tablet 78a62c4f133447e680950d0bc163f1c3 (table txn_manager_fallback_rolling [id=fad88b9f6aa644e8a17e7236195459be])
02:59:38.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.011072 21119 catalog_manager.cc:1535] Initializing Kudu cluster ID...
02:59:38.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.011860 21119 catalog_manager.cc:1310] Loaded cluster ID: c9d1ae9ce84b4ca9b3bcaf9750cc5c21
02:59:38.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.011940 21119 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
02:59:38.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.013114 21119 catalog_manager.cc:1555] Loading token signing keys...
02:59:38.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.013886 21119 catalog_manager.cc:6248] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0: Loaded TSK: 0
02:59:38.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.014691 21119 catalog_manager.cc:1565] Initializing in-progress tserver states...
02:59:38.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:38.021 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:38.021 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:38.022 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:38.023 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
02:59:38.024 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [64ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [244ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, deferred=Deferred@1955365951(state=PENDING, result=null, callback=(continuation of Deferred@1676670028 after retry RPC after error@1997128533) -> (continuation of Deferred@1045373944 after retry RPC after error@779281778) -> (continuation of Deferred@841169252 after retry RPC after error@193546014) -> (continuation of Deferred@1277093236 after retry RPC after error@1979629808) -> (continuation of Deferred@1581068082 after retry RPC after error@525152860) -> (continuation of Deferred@616776068 after retry RPC after error@658107564) -> (continuation of Deferred@273286086 after retry RPC after error@1726159937) -> (continuation of Deferred@1486007928 after retry RPC after error@816215370), errback=(continuation of Deferred@1676670028 after retry RPC after error@1997128533) -> (continuation of Deferred@1045373944 after retry RPC after error@779281778) -> (continuation of Deferred@841169252 after retry RPC after error@193546014) -> (continuation of Deferred@1277093236 after retry RPC after error@1979629808) -> (continuation of Deferred@1581068082 after retry RPC after error@525152860) -> (continuation of Deferred@616776068 after retry RPC after error@658107564) -> (continuation of Deferred@273286086 after retry RPC after error@1726159937) -> (continuation of Deferred@1486007928 after retry RPC after error@816215370))) after lookup
02:59:38.040 [DEBUG - pool-40-thread-1] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=10, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [64ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [244ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: OK, [246ms] sending RPC to server master-127.19.203.254:46111, deferred=Deferred@1955365951(state=PENDING, result=null, callback=(continuation of Deferred@1676670028 after retry RPC after error@1997128533) -> (continuation of Deferred@1045373944 after retry RPC after error@779281778) -> (continuation of Deferred@841169252 after retry RPC after error@193546014) -> (continuation of Deferred@1277093236 after retry RPC after error@1979629808) -> (continuation of Deferred@1581068082 after retry RPC after error@525152860) -> (continuation of Deferred@616776068 after retry RPC after error@658107564) -> (continuation of Deferred@273286086 after retry RPC after error@1726159937) -> (continuation of Deferred@1486007928 after retry RPC after error@816215370) -> (continuation of Deferred@926903244 after retry RPC@406955621), errback=(continuation of Deferred@1676670028 after retry RPC after error@1997128533) -> (continuation of Deferred@1045373944 after retry RPC after error@779281778) -> (continuation of Deferred@841169252 after retry RPC after error@193546014) -> (continuation of Deferred@1277093236 after retry RPC after error@1979629808) -> (continuation of Deferred@1581068082 after retry RPC after error@525152860) -> (continuation of Deferred@616776068 after retry RPC after error@658107564) -> (continuation of Deferred@273286086 after retry RPC after error@1726159937) -> (continuation of Deferred@1486007928 after retry RPC after error@816215370) -> (continuation of Deferred@926903244 after retry RPC@406955621)))
02:59:38.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:38.049300 20343 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:39212 (request call id 16) took 23 ms (client timeout 5 ms). Trace:
02:59:38.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:38.049543 20343 rpcz_store.cc:269] 1121 02:59:38.025649 (+ 0us) service_pool.cc:167] Inserting onto call queue
02:59:38.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1121 02:59:38.025744 (+ 95us) service_pool.cc:224] Handling call
02:59:38.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1121 02:59:38.049172 (+ 23428us) inbound_call.cc:177] Queueing success response
02:59:38.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"init.queue_time_us":501,"thread_start_us":2259,"threads_started":5}
02:59:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28: runtime error: signed integer overflow: 542808502342 + 9223371494046274790 cannot be represented in type 'long'
02:59:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:38.063066 21005 heartbeater.cc:648] Failed to heartbeat to 127.19.203.252:40563 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:38.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:38.068111 20864 heartbeater.cc:648] Failed to heartbeat to 127.19.203.252:40563 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:38.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:38.076351 20723 heartbeater.cc:648] Failed to heartbeat to 127.19.203.252:40563 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:38.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.112983 20311 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:49072:
02:59:38.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
02:59:38.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
02:59:38.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
02:59:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:59:38.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
02:59:38.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:38.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:38.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:59:38.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
02:59:38.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:38.115927 20311 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
02:59:38.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:38.515123 20300 consensus_peers.cc:597] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 -> Peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Couldn't send request to peer efb4c69b20bd4f9692d496c33ddbb24e. Status: Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
02:59:38.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.561162 20282 master_runner.cc:187] Error getting master registration for 127.19.203.252:40563: Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111),
02:59:38.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:38.561553 20282 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:59:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:39.010658 20300 consensus_peers.cc:597] T 00000000000000000000000000000000 P 56099a6022da440a81f20e6fe8903fb0 -> Peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Couldn't send request to peer efb4c69b20bd4f9692d496c33ddbb24e. Status: Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
02:59:39.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:39.053742 21006 heartbeater.cc:500] Master 127.19.203.254:46111 was elected leader, sending a full tablet report...
02:59:39.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:39.056548 20724 heartbeater.cc:500] Master 127.19.203.254:46111 was elected leader, sending a full tablet report...
02:59:39.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:39.066466 20865 heartbeater.cc:500] Master 127.19.203.254:46111 was elected leader, sending a full tablet report...
02:59:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 0x7fe5fdb75918 in kudu::operator+(kudu::MonoTime const&, kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28
02:59:39.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 0x7fe6143b8835 in kudu::CountDownLatch::WaitFor(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/countdown_latch.h:101:38
02:59:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 0x7fe611cf8902 in kudu::Promise<kudu::Status>::WaitFor(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/promise.h:57:16
02:59:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 0x7fe611ce8114 in kudu::master::Master::WaitForTxnManagerInit(kudu::MonoTime) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/master.cc:506:48
02:59:39.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 0x7fe611e5417e in kudu::transactions::TxnManager::CheckInitialized(kudu::MonoTime const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager.cc:283:21
02:59:39.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 0x7fe611e54d5b in kudu::transactions::TxnManager::KeepTransactionAlive(long, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, kudu::MonoTime const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager.cc:226:3
02:59:39.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 0x7fe611e59c9f in kudu::transactions::TxnManagerServiceImpl::KeepTransactionAlive(kudu::transactions::KeepTransactionAliveRequestPB const*, kudu::transactions::KeepTransactionAliveResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager_service.cc:159:42
02:59:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 0x7fe60ba9d276 in kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/asan/src/kudu/master/txn_manager.service.cc:271:13
02:59:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 0x7fe60ba9cfa5 in std::_Function_handler<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*), kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9>::_M_invoke(std::_Any_data const&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
02:59:39.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 0x7fe604d698de in std::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const ../../../include/c++/7.5.0/bits/std_function.h:706:14
02:59:39.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 0x7fe604d687be in kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3
02:59:39.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 0x7fe604d6e9f8 in kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:228:15
02:59:39.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 0x7fe604d6f35f in kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:91:5
02:59:39.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 0x7fe604d6f188 in std::_Function_handler<void (), kudu::rpc::ServicePool::Init(int)::$_0>::_M_invoke(std::_Any_data const&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
02:59:39.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 0x372c7c in std::function<void ()>::operator()() const ../../../include/c++/7.5.0/bits/std_function.h:706:14
02:59:39.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 0x7fe5fdc3add8 in kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3
02:59:39.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 0x7fe6013236da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da)
02:59:39.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 0x7fe5f9d8461e in clone (/lib/x86_64-linux-gnu/libc.so.6+0x12161e)
02:59:39.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:59:39.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28 in
02:59:39.277 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.19.203.254:46111(127.19.203.254:46111)] handling channelInactive
02:59:39.277 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state READY due to: connection closed
02:59:39.277 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.19.203.254:46111(127.19.203.254:46111) for tablet Kudu Master: connection closed
02:59:39.281 [WARN - pool-40-thread-1] (HashedWheelTimer.java:729) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
02:59:39.299 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.299 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.300 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.300 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.300 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.300 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.300 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.300 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.300 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.320 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.320 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.321 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.321 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.321 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.321 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.341 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.341 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.341 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.341 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.342 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.342 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.342 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.342 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.361 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.361 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.361 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.361 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.361 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.362 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.362 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.381 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.381 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.381 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.382 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.382 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.382 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.382 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.382 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.421 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.421 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.421 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.421 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.421 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.421 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.421 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.422 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:39.439837 21141 raft_consensus.cc:534] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 56099a6022da440a81f20e6fe8903fb0)
02:59:39.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:39.440093 21141 raft_consensus.cc:556] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:39.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.441 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:39.441077 21141 leader_election.cc:290] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
02:59:39.441 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.441 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.442 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.442 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.442 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.442 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:39.442698 20378 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.203.254:46111: connect: Connection refused (error 111) [suppressed 6 similar messages]
02:59:39.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:39.444789 20381 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:39.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:39.445019 20378 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111): Network error: Client connection negotiation failed: client connection to 127.19.203.254:46111: connect: Connection refused (error 111)
02:59:39.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:39.445171 20378 leader_election.cc:304] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c; no voters: 56099a6022da440a81f20e6fe8903fb0, efb4c69b20bd4f9692d496c33ddbb24e
02:59:39.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:39.445449 21141 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:59:39.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.482 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.482 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.482 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.482 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.483 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.484 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.484 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.484 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.501 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.501 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.501 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.501 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.502 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.502 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.502 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.699 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.701 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.701 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.701 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.701 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.701 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.701 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.721 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.721 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.721 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.721 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.722 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.722 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.722 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.722 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.741 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.741 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.741 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.741 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.741 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.741 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.761 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.761 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.761 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.762 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.762 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.762 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.762 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.781 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.781 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.781 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.781 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.781 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.781 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.782 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.782 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.801 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.801 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.801 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.801 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.801 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.801 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.841 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.841 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.841 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.842 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.842 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.842 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.842 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.842 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.842 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.842 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.843 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.843 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=143), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] refreshing cache from master, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [144ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1269096918(state=PENDING, result=null, callback=(continuation of Deferred@1888485074 after retry RPC after error@1344895161) -> (continuation of Deferred@892565327 after retry RPC after error@1509268078) -> (continuation of Deferred@1451464428 after retry RPC after error@1307889717) -> (continuation of Deferred@1666408395 after retry RPC after error@369139493) -> (continuation of Deferred@1560174591 after retry RPC after error@929057069) -> (continuation of Deferred@1312089357 after retry RPC after error@1558164057), errback=(continuation of Deferred@1888485074 after retry RPC after error@1344895161) -> (continuation of Deferred@892565327 after retry RPC after error@1509268078) -> (continuation of Deferred@1451464428 after retry RPC after error@1307889717) -> (continuation of Deferred@1666408395 after retry RPC after error@369139493) -> (continuation of Deferred@1560174591 after retry RPC after error@929057069) -> (continuation of Deferred@1312089357 after retry RPC after error@1558164057)))
02:59:39.843 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=143), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] refreshing cache from master, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [144ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1269096918(state=PENDING, result=null, callback=(continuation of Deferred@1888485074 after retry RPC after error@1344895161) -> (continuation of Deferred@892565327 after retry RPC after error@1509268078) -> (continuation of Deferred@1451464428 after retry RPC after error@1307889717) -> (continuation of Deferred@1666408395 after retry RPC after error@369139493) -> (continuation of Deferred@1560174591 after retry RPC after error@929057069) -> (continuation of Deferred@1312089357 after retry RPC after error@1558164057), errback=(continuation of Deferred@1888485074 after retry RPC after error@1344895161) -> (continuation of Deferred@892565327 after retry RPC after error@1509268078) -> (continuation of Deferred@1451464428 after retry RPC after error@1307889717) -> (continuation of Deferred@1666408395 after retry RPC after error@369139493) -> (continuation of Deferred@1560174591 after retry RPC after error@929057069) -> (continuation of Deferred@1312089357 after retry RPC after error@1558164057)))
02:59:39.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:39.847880 21141 raft_consensus.cc:534] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 56099a6022da440a81f20e6fe8903fb0)
02:59:39.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:39.848093 21141 raft_consensus.cc:556] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:39.848842 21141 leader_election.cc:290] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
02:59:39.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:39.850615 20381 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:39.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:39.850778 20378 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111): Network error: Client connection negotiation failed: client connection to 127.19.203.254:46111: connect: Connection refused (error 111)
02:59:39.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:39.850917 20378 leader_election.cc:304] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c; no voters: 56099a6022da440a81f20e6fe8903fb0, efb4c69b20bd4f9692d496c33ddbb24e
02:59:39.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:39.851219 21141 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:59:39.861 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.861 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.862 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.862 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.862 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.862 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.862 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.862 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.862 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.881 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.881 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.882 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.882 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.882 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.882 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.882 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.882 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.883 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.901 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.901 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.901 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.902 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.902 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.902 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.902 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.902 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.921 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.921 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.921 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.921 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.921 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.922 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.922 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.941 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.941 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.941 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.941 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.941 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.941 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.942 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.942 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:39.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:39.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:39.981 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.981 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.981 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:39.981 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.981 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.981 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:39.982 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.001 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.001 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.001 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.001 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.001 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.001 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.002 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.002 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.021 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.021 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.021 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.021 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.021 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.021 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.021 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.022 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@864857915(state=PENDING, result=null, callback=(continuation of Deferred@1628214327 after retry RPC after error@1770914176) -> (continuation of Deferred@102927299 after retry RPC after error@1342245195) -> (continuation of Deferred@1428787792 after retry RPC after error@1286740852) -> (continuation of Deferred@1591541170 after retry RPC after error@105901171) -> (continuation of Deferred@535647736 after retry RPC after error@1129878730) -> (continuation of Deferred@2060501118 after retry RPC after error@50693923), errback=(continuation of Deferred@1628214327 after retry RPC after error@1770914176) -> (continuation of Deferred@102927299 after retry RPC after error@1342245195) -> (continuation of Deferred@1428787792 after retry RPC after error@1286740852) -> (continuation of Deferred@1591541170 after retry RPC after error@105901171) -> (continuation of Deferred@535647736 after retry RPC after error@1129878730) -> (continuation of Deferred@2060501118 after retry RPC after error@50693923)))
02:59:40.022 [DEBUG - kudu-nio-2] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@864857915(state=PENDING, result=null, callback=(continuation of Deferred@1628214327 after retry RPC after error@1770914176) -> (continuation of Deferred@102927299 after retry RPC after error@1342245195) -> (continuation of Deferred@1428787792 after retry RPC after error@1286740852) -> (continuation of Deferred@1591541170 after retry RPC after error@105901171) -> (continuation of Deferred@535647736 after retry RPC after error@1129878730) -> (continuation of Deferred@2060501118 after retry RPC after error@50693923), errback=(continuation of Deferred@1628214327 after retry RPC after error@1770914176) -> (continuation of Deferred@102927299 after retry RPC after error@1342245195) -> (continuation of Deferred@1428787792 after retry RPC after error@1286740852) -> (continuation of Deferred@1591541170 after retry RPC after error@105901171) -> (continuation of Deferred@535647736 after retry RPC after error@1129878730) -> (continuation of Deferred@2060501118 after retry RPC after error@50693923)))
02:59:40.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.041 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.041 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.041 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.041 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.041 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.041 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.042 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.042 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.061 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.061 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.061 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.062 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.063 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.063 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.064 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.064 [WARN - pool-40-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.081 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.081 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.081 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.081 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.081 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.081 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.082 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.101 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.101 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.101 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.101 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.101 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.102 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.102 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.102 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.121 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.121 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.121 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.121 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.121 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.121 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.161 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.161 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.161 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.162 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.162 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.162 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.162 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.162 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.221 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.221 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.221 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.221 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.221 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.222 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.222 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.222 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.223 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [24ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [24ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@949735035(state=PENDING, result=null, callback=(continuation of Deferred@379393346 after retry RPC after error@721945768) -> (continuation of Deferred@453206979 after retry RPC after error@2106429134) -> (continuation of Deferred@63218488 after retry RPC after error@1753406124) -> (continuation of Deferred@1746903300 after retry RPC after error@2037754167) -> (continuation of Deferred@1594755138 after retry RPC after error@564653749) -> (continuation of Deferred@874867716 after retry RPC after error@1615677350), errback=(continuation of Deferred@379393346 after retry RPC after error@721945768) -> (continuation of Deferred@453206979 after retry RPC after error@2106429134) -> (continuation of Deferred@63218488 after retry RPC after error@1753406124) -> (continuation of Deferred@1746903300 after retry RPC after error@2037754167) -> (continuation of Deferred@1594755138 after retry RPC after error@564653749) -> (continuation of Deferred@874867716 after retry RPC after error@1615677350)))
02:59:40.223 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [24ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [24ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@949735035(state=PENDING, result=null, callback=(continuation of Deferred@379393346 after retry RPC after error@721945768) -> (continuation of Deferred@453206979 after retry RPC after error@2106429134) -> (continuation of Deferred@63218488 after retry RPC after error@1753406124) -> (continuation of Deferred@1746903300 after retry RPC after error@2037754167) -> (continuation of Deferred@1594755138 after retry RPC after error@564653749) -> (continuation of Deferred@874867716 after retry RPC after error@1615677350), errback=(continuation of Deferred@379393346 after retry RPC after error@721945768) -> (continuation of Deferred@453206979 after retry RPC after error@2106429134) -> (continuation of Deferred@63218488 after retry RPC after error@1753406124) -> (continuation of Deferred@1746903300 after retry RPC after error@2037754167) -> (continuation of Deferred@1594755138 after retry RPC after error@564653749) -> (continuation of Deferred@874867716 after retry RPC after error@1615677350)))
02:59:40.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.241 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.241 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.241 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.241 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.241 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.241 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.242 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.261 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.261 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.261 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.261 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.262 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.262 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.262 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.262 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.281 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.281 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.281 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.281 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.281 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.281 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.301 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.301 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.301 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.301 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.301 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.301 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.301 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.302 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.321 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.321 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.321 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.321 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.321 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.321 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.322 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:40.333037 21141 raft_consensus.cc:534] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 56099a6022da440a81f20e6fe8903fb0)
02:59:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:40.333289 21141 raft_consensus.cc:556] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:40.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:40.334043 21141 leader_election.cc:290] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
02:59:40.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:40.336305 20378 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111): Network error: Client connection negotiation failed: client connection to 127.19.203.254:46111: connect: Connection refused (error 111)
02:59:40.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:40.336915 20381 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:40.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:40.337071 20381 leader_election.cc:304] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c; no voters: 56099a6022da440a81f20e6fe8903fb0, efb4c69b20bd4f9692d496c33ddbb24e
02:59:40.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:40.337844 21141 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:59:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:40.341841 20605 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.150s from last keepalive heartbeat (effective timeout is 1.000s)
02:59:40.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.361 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.361 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.361 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.361 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.361 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.361 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.362 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.362 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.381 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.381 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.381 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.381 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.381 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.381 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.481 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.481 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.481 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.481 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.481 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.481 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.482 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.483 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [3ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2074931547(state=PENDING, result=null, callback=(continuation of Deferred@1042022743 after retry RPC after error@883039162) -> (continuation of Deferred@1682711736 after retry RPC after error@2029401923) -> (continuation of Deferred@641636275 after retry RPC after error@40187653) -> (continuation of Deferred@126462485 after retry RPC after error@1309460872) -> (continuation of Deferred@484269301 after retry RPC after error@933989748) -> (continuation of Deferred@2100842646 after retry RPC after error@1439733296) -> (continuation of Deferred@597443211 after retry RPC after error@1211612065), errback=(continuation of Deferred@1042022743 after retry RPC after error@883039162) -> (continuation of Deferred@1682711736 after retry RPC after error@2029401923) -> (continuation of Deferred@641636275 after retry RPC after error@40187653) -> (continuation of Deferred@126462485 after retry RPC after error@1309460872) -> (continuation of Deferred@484269301 after retry RPC after error@933989748) -> (continuation of Deferred@2100842646 after retry RPC after error@1439733296) -> (continuation of Deferred@597443211 after retry RPC after error@1211612065)))
02:59:40.483 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [3ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2074931547(state=PENDING, result=null, callback=(continuation of Deferred@1042022743 after retry RPC after error@883039162) -> (continuation of Deferred@1682711736 after retry RPC after error@2029401923) -> (continuation of Deferred@641636275 after retry RPC after error@40187653) -> (continuation of Deferred@126462485 after retry RPC after error@1309460872) -> (continuation of Deferred@484269301 after retry RPC after error@933989748) -> (continuation of Deferred@2100842646 after retry RPC after error@1439733296) -> (continuation of Deferred@597443211 after retry RPC after error@1211612065), errback=(continuation of Deferred@1042022743 after retry RPC after error@883039162) -> (continuation of Deferred@1682711736 after retry RPC after error@2029401923) -> (continuation of Deferred@641636275 after retry RPC after error@40187653) -> (continuation of Deferred@126462485 after retry RPC after error@1309460872) -> (continuation of Deferred@484269301 after retry RPC after error@933989748) -> (continuation of Deferred@2100842646 after retry RPC after error@1439733296) -> (continuation of Deferred@597443211 after retry RPC after error@1211612065)))
02:59:40.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.501 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.501 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.501 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.501 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.501 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.501 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.502 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.503 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.521 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.521 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.521 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.521 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.522 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.522 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.522 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.522 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.541 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.541 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.541 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.541 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.542 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.542 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.542 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.543 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.561 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.561 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.561 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.561 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.562 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.562 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.562 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.562 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.581 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.581 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.581 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.581 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.581 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.581 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.601 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.601 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.601 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.601 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.601 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.601 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.602 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.602 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.641 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.641 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.641 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.641 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.642 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.642 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.642 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.642 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.642 [DEBUG - kudu-nio-6] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [3ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@749671505(state=PENDING, result=null, callback=(continuation of Deferred@825367846 after retry RPC after error@1501749534) -> (continuation of Deferred@819505576 after retry RPC after error@1677799038) -> (continuation of Deferred@1165743349 after retry RPC after error@188700501) -> (continuation of Deferred@2128554611 after retry RPC after error@331385096) -> (continuation of Deferred@134687737 after retry RPC after error@2037600339) -> (continuation of Deferred@2121822652 after retry RPC after error@1776762802), errback=(continuation of Deferred@825367846 after retry RPC after error@1501749534) -> (continuation of Deferred@819505576 after retry RPC after error@1677799038) -> (continuation of Deferred@1165743349 after retry RPC after error@188700501) -> (continuation of Deferred@2128554611 after retry RPC after error@331385096) -> (continuation of Deferred@134687737 after retry RPC after error@2037600339) -> (continuation of Deferred@2121822652 after retry RPC after error@1776762802)))
02:59:40.643 [DEBUG - kudu-nio-6] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [3ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@749671505(state=PENDING, result=null, callback=(continuation of Deferred@825367846 after retry RPC after error@1501749534) -> (continuation of Deferred@819505576 after retry RPC after error@1677799038) -> (continuation of Deferred@1165743349 after retry RPC after error@188700501) -> (continuation of Deferred@2128554611 after retry RPC after error@331385096) -> (continuation of Deferred@134687737 after retry RPC after error@2037600339) -> (continuation of Deferred@2121822652 after retry RPC after error@1776762802), errback=(continuation of Deferred@825367846 after retry RPC after error@1501749534) -> (continuation of Deferred@819505576 after retry RPC after error@1677799038) -> (continuation of Deferred@1165743349 after retry RPC after error@188700501) -> (continuation of Deferred@2128554611 after retry RPC after error@331385096) -> (continuation of Deferred@134687737 after retry RPC after error@2037600339) -> (continuation of Deferred@2121822652 after retry RPC after error@1776762802)))
02:59:40.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.661 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.661 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.662 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.662 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.662 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.662 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.663 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.663 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.681 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.681 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.681 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.681 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.682 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.682 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.682 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.701 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.701 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.701 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.701 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.701 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.701 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.701 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.702 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.719 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.720 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.720 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.720 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.720 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.721 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.721 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.721 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.741 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.741 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.741 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.741 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.741 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.741 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.742 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.781 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.781 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.781 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.781 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.781 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.782 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.782 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.782 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.821 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.821 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.821 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.821 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.821 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.821 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.822 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1328977742(state=PENDING, result=null, callback=(continuation of Deferred@1338107167 after retry RPC after error@181861475) -> (continuation of Deferred@1012269680 after retry RPC after error@1201765971) -> (continuation of Deferred@118002341 after retry RPC after error@973099971) -> (continuation of Deferred@910787441 after retry RPC after error@2060853261) -> (continuation of Deferred@965216328 after retry RPC after error@28232433) -> (continuation of Deferred@2064028606 after retry RPC after error@807080603), errback=(continuation of Deferred@1338107167 after retry RPC after error@181861475) -> (continuation of Deferred@1012269680 after retry RPC after error@1201765971) -> (continuation of Deferred@118002341 after retry RPC after error@973099971) -> (continuation of Deferred@910787441 after retry RPC after error@2060853261) -> (continuation of Deferred@965216328 after retry RPC after error@28232433) -> (continuation of Deferred@2064028606 after retry RPC after error@807080603)))
02:59:40.822 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1328977742(state=PENDING, result=null, callback=(continuation of Deferred@1338107167 after retry RPC after error@181861475) -> (continuation of Deferred@1012269680 after retry RPC after error@1201765971) -> (continuation of Deferred@118002341 after retry RPC after error@973099971) -> (continuation of Deferred@910787441 after retry RPC after error@2060853261) -> (continuation of Deferred@965216328 after retry RPC after error@28232433) -> (continuation of Deferred@2064028606 after retry RPC after error@807080603), errback=(continuation of Deferred@1338107167 after retry RPC after error@181861475) -> (continuation of Deferred@1012269680 after retry RPC after error@1201765971) -> (continuation of Deferred@118002341 after retry RPC after error@973099971) -> (continuation of Deferred@910787441 after retry RPC after error@2060853261) -> (continuation of Deferred@965216328 after retry RPC after error@28232433) -> (continuation of Deferred@2064028606 after retry RPC after error@807080603)))
02:59:40.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.841 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.841 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.841 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.841 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.842 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.842 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.842 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.842 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.861 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.861 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.861 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.861 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.861 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.861 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.881 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.881 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.881 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.881 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.882 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.882 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.882 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.882 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.901 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.901 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.901 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.901 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.901 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.901 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.921 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.921 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.921 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.921 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.921 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.922 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.922 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.922 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:40.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:40.961 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.961 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.961 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:40.961 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.961 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.961 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:40.962 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:40.999 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.001 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.001 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.001 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.001 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.001 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.001 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.001 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.021 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.021 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.021 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.021 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.021 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.021 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.081 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.081 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.081 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.081 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.081 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.081 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.081 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.082 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@344213908(state=PENDING, result=null, callback=(continuation of Deferred@453839182 after retry RPC after error@2093011915) -> (continuation of Deferred@1239620331 after retry RPC after error@2132619946) -> (continuation of Deferred@212403070 after retry RPC after error@1989003179) -> (continuation of Deferred@669560787 after retry RPC after error@838226406) -> (continuation of Deferred@1971328597 after retry RPC after error@1839736785) -> (continuation of Deferred@266111872 after retry RPC after error@203850492) -> (continuation of Deferred@2044792723 after retry RPC after error@1432037841) -> (continuation of Deferred@1051736177 after retry RPC after error@112819257), errback=(continuation of Deferred@453839182 after retry RPC after error@2093011915) -> (continuation of Deferred@1239620331 after retry RPC after error@2132619946) -> (continuation of Deferred@212403070 after retry RPC after error@1989003179) -> (continuation of Deferred@669560787 after retry RPC after error@838226406) -> (continuation of Deferred@1971328597 after retry RPC after error@1839736785) -> (continuation of Deferred@266111872 after retry RPC after error@203850492) -> (continuation of Deferred@2044792723 after retry RPC after error@1432037841) -> (continuation of Deferred@1051736177 after retry RPC after error@112819257)))
02:59:41.082 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@344213908(state=PENDING, result=null, callback=(continuation of Deferred@453839182 after retry RPC after error@2093011915) -> (continuation of Deferred@1239620331 after retry RPC after error@2132619946) -> (continuation of Deferred@212403070 after retry RPC after error@1989003179) -> (continuation of Deferred@669560787 after retry RPC after error@838226406) -> (continuation of Deferred@1971328597 after retry RPC after error@1839736785) -> (continuation of Deferred@266111872 after retry RPC after error@203850492) -> (continuation of Deferred@2044792723 after retry RPC after error@1432037841) -> (continuation of Deferred@1051736177 after retry RPC after error@112819257), errback=(continuation of Deferred@453839182 after retry RPC after error@2093011915) -> (continuation of Deferred@1239620331 after retry RPC after error@2132619946) -> (continuation of Deferred@212403070 after retry RPC after error@1989003179) -> (continuation of Deferred@669560787 after retry RPC after error@838226406) -> (continuation of Deferred@1971328597 after retry RPC after error@1839736785) -> (continuation of Deferred@266111872 after retry RPC after error@203850492) -> (continuation of Deferred@2044792723 after retry RPC after error@1432037841) -> (continuation of Deferred@1051736177 after retry RPC after error@112819257)))
02:59:41.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.101 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.101 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.101 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.101 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.101 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.101 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.102 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.102 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.121 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.121 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.121 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.121 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.121 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.121 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.141 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.141 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.141 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.141 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.141 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.141 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.142 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.159 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.160 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.160 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.160 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.161 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.161 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.161 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.181 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.181 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.181 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.181 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.181 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.181 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.182 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.221 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.221 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.221 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.221 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.222 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.222 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.222 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.222 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.299 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.300 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.301 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.301 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.301 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.301 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.301 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:41.312207 21166 raft_consensus.cc:534] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 56099a6022da440a81f20e6fe8903fb0)
02:59:41.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:41.312471 21166 raft_consensus.cc:556] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:41.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:41.313354 21166 leader_election.cc:290] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
02:59:41.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:41.315750 20378 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111): Network error: Client connection negotiation failed: client connection to 127.19.203.254:46111: connect: Connection refused (error 111)
02:59:41.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:41.315964 20381 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:41.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:41.316107 20381 leader_election.cc:304] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c; no voters: 56099a6022da440a81f20e6fe8903fb0, efb4c69b20bd4f9692d496c33ddbb24e
02:59:41.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:41.316357 21166 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:59:41.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.341 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.341 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.341 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.341 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.341 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.341 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.341 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.342 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.342 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1437278507(state=PENDING, result=null, callback=(continuation of Deferred@1218037794 after retry RPC after error@215680825) -> (continuation of Deferred@709818811 after retry RPC after error@643835808) -> (continuation of Deferred@549335958 after retry RPC after error@1946684023) -> (continuation of Deferred@701937260 after retry RPC after error@237820369) -> (continuation of Deferred@1600092417 after retry RPC after error@1806736175) -> (continuation of Deferred@2009675235 after retry RPC after error@524523674) -> (continuation of Deferred@997223682 after retry RPC after error@1417076896), errback=(continuation of Deferred@1218037794 after retry RPC after error@215680825) -> (continuation of Deferred@709818811 after retry RPC after error@643835808) -> (continuation of Deferred@549335958 after retry RPC after error@1946684023) -> (continuation of Deferred@701937260 after retry RPC after error@237820369) -> (continuation of Deferred@1600092417 after retry RPC after error@1806736175) -> (continuation of Deferred@2009675235 after retry RPC after error@524523674) -> (continuation of Deferred@997223682 after retry RPC after error@1417076896)))
02:59:41.342 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1437278507(state=PENDING, result=null, callback=(continuation of Deferred@1218037794 after retry RPC after error@215680825) -> (continuation of Deferred@709818811 after retry RPC after error@643835808) -> (continuation of Deferred@549335958 after retry RPC after error@1946684023) -> (continuation of Deferred@701937260 after retry RPC after error@237820369) -> (continuation of Deferred@1600092417 after retry RPC after error@1806736175) -> (continuation of Deferred@2009675235 after retry RPC after error@524523674) -> (continuation of Deferred@997223682 after retry RPC after error@1417076896), errback=(continuation of Deferred@1218037794 after retry RPC after error@215680825) -> (continuation of Deferred@709818811 after retry RPC after error@643835808) -> (continuation of Deferred@549335958 after retry RPC after error@1946684023) -> (continuation of Deferred@701937260 after retry RPC after error@237820369) -> (continuation of Deferred@1600092417 after retry RPC after error@1806736175) -> (continuation of Deferred@2009675235 after retry RPC after error@524523674) -> (continuation of Deferred@997223682 after retry RPC after error@1417076896)))
02:59:41.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.361 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.361 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.361 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.361 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.361 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.361 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.361 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.361 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.362 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.381 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.381 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.381 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.381 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.381 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.382 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.382 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.382 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.401 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.401 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.401 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.401 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.401 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.401 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.402 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.421 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.421 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.421 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.421 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.421 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.421 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.421 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.422 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.439 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.440 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.440 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.440 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.441 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.441 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.441 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.481 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.481 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.481 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.481 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.481 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.481 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.482 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.501 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.501 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.501 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.501 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.501 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.502 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.502 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.502 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.601 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.601 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.601 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.601 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.602 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.602 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.602 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.602 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.603 [DEBUG - kudu-nio-6] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1389667392(state=PENDING, result=null, callback=(continuation of Deferred@85510506 after retry RPC after error@1804537141) -> (continuation of Deferred@880201675 after retry RPC after error@1140300705) -> (continuation of Deferred@6773036 after retry RPC after error@319949488) -> (continuation of Deferred@295212942 after retry RPC after error@1850555535) -> (continuation of Deferred@1415161427 after retry RPC after error@1469936098) -> (continuation of Deferred@49068152 after retry RPC after error@88233454) -> (continuation of Deferred@472907819 after retry RPC after error@547254318), errback=(continuation of Deferred@85510506 after retry RPC after error@1804537141) -> (continuation of Deferred@880201675 after retry RPC after error@1140300705) -> (continuation of Deferred@6773036 after retry RPC after error@319949488) -> (continuation of Deferred@295212942 after retry RPC after error@1850555535) -> (continuation of Deferred@1415161427 after retry RPC after error@1469936098) -> (continuation of Deferred@49068152 after retry RPC after error@88233454) -> (continuation of Deferred@472907819 after retry RPC after error@547254318)))
02:59:41.603 [DEBUG - kudu-nio-6] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1389667392(state=PENDING, result=null, callback=(continuation of Deferred@85510506 after retry RPC after error@1804537141) -> (continuation of Deferred@880201675 after retry RPC after error@1140300705) -> (continuation of Deferred@6773036 after retry RPC after error@319949488) -> (continuation of Deferred@295212942 after retry RPC after error@1850555535) -> (continuation of Deferred@1415161427 after retry RPC after error@1469936098) -> (continuation of Deferred@49068152 after retry RPC after error@88233454) -> (continuation of Deferred@472907819 after retry RPC after error@547254318), errback=(continuation of Deferred@85510506 after retry RPC after error@1804537141) -> (continuation of Deferred@880201675 after retry RPC after error@1140300705) -> (continuation of Deferred@6773036 after retry RPC after error@319949488) -> (continuation of Deferred@295212942 after retry RPC after error@1850555535) -> (continuation of Deferred@1415161427 after retry RPC after error@1469936098) -> (continuation of Deferred@49068152 after retry RPC after error@88233454) -> (continuation of Deferred@472907819 after retry RPC after error@547254318)))
02:59:41.619 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.621 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.621 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.621 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.621 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.621 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.621 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.641 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.641 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.641 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.641 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.641 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.641 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.661 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.661 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.661 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.661 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.661 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.661 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.661 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.663 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.679 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.681 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.681 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.681 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.681 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.681 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.681 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.681 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.701 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.701 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.701 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.701 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.701 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.702 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.702 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.702 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.741 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.741 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.741 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.741 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.741 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.741 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:41.785213 21166 raft_consensus.cc:534] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 56099a6022da440a81f20e6fe8903fb0)
02:59:41.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:41.785419 21166 raft_consensus.cc:556] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:41.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:41.786150 21166 leader_election.cc:290] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
02:59:41.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:41.788089 20378 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111): Network error: Client connection negotiation failed: client connection to 127.19.203.254:46111: connect: Connection refused (error 111)
02:59:41.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:41.788293 20381 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:41.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:41.788436 20381 leader_election.cc:304] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c; no voters: 56099a6022da440a81f20e6fe8903fb0, efb4c69b20bd4f9692d496c33ddbb24e
02:59:41.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:41.788877 21166 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:59:41.819 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.820 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.820 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.820 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.821 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.821 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.821 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.821 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1833958758(state=PENDING, result=null, callback=(continuation of Deferred@12171794 after retry RPC after error@533930588) -> (continuation of Deferred@632085636 after retry RPC after error@924496282) -> (continuation of Deferred@2059334721 after retry RPC after error@2088364168) -> (continuation of Deferred@1064452991 after retry RPC after error@1560716138) -> (continuation of Deferred@1343703159 after retry RPC after error@1662637909) -> (continuation of Deferred@1314720242 after retry RPC after error@640216736), errback=(continuation of Deferred@12171794 after retry RPC after error@533930588) -> (continuation of Deferred@632085636 after retry RPC after error@924496282) -> (continuation of Deferred@2059334721 after retry RPC after error@2088364168) -> (continuation of Deferred@1064452991 after retry RPC after error@1560716138) -> (continuation of Deferred@1343703159 after retry RPC after error@1662637909) -> (continuation of Deferred@1314720242 after retry RPC after error@640216736)))
02:59:41.822 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1833958758(state=PENDING, result=null, callback=(continuation of Deferred@12171794 after retry RPC after error@533930588) -> (continuation of Deferred@632085636 after retry RPC after error@924496282) -> (continuation of Deferred@2059334721 after retry RPC after error@2088364168) -> (continuation of Deferred@1064452991 after retry RPC after error@1560716138) -> (continuation of Deferred@1343703159 after retry RPC after error@1662637909) -> (continuation of Deferred@1314720242 after retry RPC after error@640216736), errback=(continuation of Deferred@12171794 after retry RPC after error@533930588) -> (continuation of Deferred@632085636 after retry RPC after error@924496282) -> (continuation of Deferred@2059334721 after retry RPC after error@2088364168) -> (continuation of Deferred@1064452991 after retry RPC after error@1560716138) -> (continuation of Deferred@1343703159 after retry RPC after error@1662637909) -> (continuation of Deferred@1314720242 after retry RPC after error@640216736)))
02:59:41.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.841 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.841 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.841 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.842 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.842 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.842 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.842 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.842 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.842 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.843 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.859 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.860 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.860 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.860 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.861 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.861 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.861 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.881 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.881 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.881 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.881 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.881 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.881 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.901 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.901 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.901 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.901 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.901 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.901 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.921 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.921 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.921 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.921 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.921 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.921 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.921 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.922 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.939 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.940 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.940 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.941 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.941 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.941 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.941 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:41.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:41.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:41.981 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.981 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.981 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:41.981 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.981 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.981 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:41.982 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.019 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.021 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.021 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.021 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.021 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.021 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.021 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.022 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=180), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [0ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [18ms] refreshing cache from master, [18ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [18ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [19ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [20ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [38ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [98ms] refreshing cache from master, [98ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [98ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [99ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [100ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [138ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [178ms] refreshing cache from master, [178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1485769943(state=PENDING, result=null, callback=(continuation of Deferred@2112392903 after retry RPC after error@1510199652) -> (continuation of Deferred@2030897250 after retry RPC after error@1497647648) -> (continuation of Deferred@1625516409 after retry RPC after error@615455984) -> (continuation of Deferred@896556575 after retry RPC after error@1310590939) -> (continuation of Deferred@902629312 after retry RPC after error@1767494933) -> (continuation of Deferred@1584328327 after retry RPC after error@1626348110) -> (continuation of Deferred@1029632663 after retry RPC after error@445147476), errback=(continuation of Deferred@2112392903 after retry RPC after error@1510199652) -> (continuation of Deferred@2030897250 after retry RPC after error@1497647648) -> (continuation of Deferred@1625516409 after retry RPC after error@615455984) -> (continuation of Deferred@896556575 after retry RPC after error@1310590939) -> (continuation of Deferred@902629312 after retry RPC after error@1767494933) -> (continuation of Deferred@1584328327 after retry RPC after error@1626348110) -> (continuation of Deferred@1029632663 after retry RPC after error@445147476)))
02:59:42.022 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=180), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [0ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [18ms] refreshing cache from master, [18ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [18ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [19ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [20ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [38ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [98ms] refreshing cache from master, [98ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [98ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [99ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [100ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [138ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [178ms] refreshing cache from master, [178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1485769943(state=PENDING, result=null, callback=(continuation of Deferred@2112392903 after retry RPC after error@1510199652) -> (continuation of Deferred@2030897250 after retry RPC after error@1497647648) -> (continuation of Deferred@1625516409 after retry RPC after error@615455984) -> (continuation of Deferred@896556575 after retry RPC after error@1310590939) -> (continuation of Deferred@902629312 after retry RPC after error@1767494933) -> (continuation of Deferred@1584328327 after retry RPC after error@1626348110) -> (continuation of Deferred@1029632663 after retry RPC after error@445147476), errback=(continuation of Deferred@2112392903 after retry RPC after error@1510199652) -> (continuation of Deferred@2030897250 after retry RPC after error@1497647648) -> (continuation of Deferred@1625516409 after retry RPC after error@615455984) -> (continuation of Deferred@896556575 after retry RPC after error@1310590939) -> (continuation of Deferred@902629312 after retry RPC after error@1767494933) -> (continuation of Deferred@1584328327 after retry RPC after error@1626348110) -> (continuation of Deferred@1029632663 after retry RPC after error@445147476)))
02:59:42.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.041 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.041 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.041 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.041 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.041 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.041 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.042 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.061 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.061 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.061 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.062 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.063 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.063 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.064 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.064 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.081 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.081 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.081 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.081 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.081 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.081 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.082 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.101 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.101 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.101 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.101 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.101 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.101 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.101 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.102 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.119 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.121 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.121 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.121 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.121 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.121 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.121 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.141 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.141 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.141 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.141 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.141 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.141 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.142 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.181 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.181 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.181 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.182 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.182 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.182 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.182 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.182 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.241 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.241 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.241 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.241 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.241 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.241 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.242 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.242 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [24ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [24ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [25ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@849350990(state=PENDING, result=null, callback=(continuation of Deferred@1612620261 after retry RPC after error@863980835) -> (continuation of Deferred@915739040 after retry RPC after error@76373085) -> (continuation of Deferred@285852832 after retry RPC after error@1490458103) -> (continuation of Deferred@1179686748 after retry RPC after error@676418687) -> (continuation of Deferred@2007148817 after retry RPC after error@456651583) -> (continuation of Deferred@1503624562 after retry RPC after error@2068789394) -> (continuation of Deferred@432532683 after retry RPC after error@1028479687), errback=(continuation of Deferred@1612620261 after retry RPC after error@863980835) -> (continuation of Deferred@915739040 after retry RPC after error@76373085) -> (continuation of Deferred@285852832 after retry RPC after error@1490458103) -> (continuation of Deferred@1179686748 after retry RPC after error@676418687) -> (continuation of Deferred@2007148817 after retry RPC after error@456651583) -> (continuation of Deferred@1503624562 after retry RPC after error@2068789394) -> (continuation of Deferred@432532683 after retry RPC after error@1028479687)))
02:59:42.242 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [24ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [24ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [25ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@849350990(state=PENDING, result=null, callback=(continuation of Deferred@1612620261 after retry RPC after error@863980835) -> (continuation of Deferred@915739040 after retry RPC after error@76373085) -> (continuation of Deferred@285852832 after retry RPC after error@1490458103) -> (continuation of Deferred@1179686748 after retry RPC after error@676418687) -> (continuation of Deferred@2007148817 after retry RPC after error@456651583) -> (continuation of Deferred@1503624562 after retry RPC after error@2068789394) -> (continuation of Deferred@432532683 after retry RPC after error@1028479687), errback=(continuation of Deferred@1612620261 after retry RPC after error@863980835) -> (continuation of Deferred@915739040 after retry RPC after error@76373085) -> (continuation of Deferred@285852832 after retry RPC after error@1490458103) -> (continuation of Deferred@1179686748 after retry RPC after error@676418687) -> (continuation of Deferred@2007148817 after retry RPC after error@456651583) -> (continuation of Deferred@1503624562 after retry RPC after error@2068789394) -> (continuation of Deferred@432532683 after retry RPC after error@1028479687)))
02:59:42.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.261 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.261 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.262 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.262 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.262 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.262 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.262 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.262 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.281 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.281 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.281 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.281 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.281 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.281 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.282 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.301 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.301 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.301 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.301 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.301 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.301 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.301 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.302 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.319 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.320 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.320 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.320 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.321 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.321 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.321 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.321 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.321 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.340 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.341 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.341 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.341 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.341 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.341 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.381 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.381 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.381 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.381 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.381 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.382 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.382 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.382 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.421 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.421 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.421 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.421 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.421 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.421 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.422 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.479 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.480 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.480 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.480 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.481 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.481 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.481 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.482 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.482 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@765784340(state=PENDING, result=null, callback=(continuation of Deferred@656237356 after retry RPC after error@864792416) -> (continuation of Deferred@395808175 after retry RPC after error@498201112) -> (continuation of Deferred@1375336065 after retry RPC after error@1072145265) -> (continuation of Deferred@347519959 after retry RPC after error@101034603) -> (continuation of Deferred@416680404 after retry RPC after error@447274388) -> (continuation of Deferred@996254997 after retry RPC after error@278660832) -> (continuation of Deferred@950805902 after retry RPC after error@1681136498), errback=(continuation of Deferred@656237356 after retry RPC after error@864792416) -> (continuation of Deferred@395808175 after retry RPC after error@498201112) -> (continuation of Deferred@1375336065 after retry RPC after error@1072145265) -> (continuation of Deferred@347519959 after retry RPC after error@101034603) -> (continuation of Deferred@416680404 after retry RPC after error@447274388) -> (continuation of Deferred@996254997 after retry RPC after error@278660832) -> (continuation of Deferred@950805902 after retry RPC after error@1681136498)))
02:59:42.482 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@765784340(state=PENDING, result=null, callback=(continuation of Deferred@656237356 after retry RPC after error@864792416) -> (continuation of Deferred@395808175 after retry RPC after error@498201112) -> (continuation of Deferred@1375336065 after retry RPC after error@1072145265) -> (continuation of Deferred@347519959 after retry RPC after error@101034603) -> (continuation of Deferred@416680404 after retry RPC after error@447274388) -> (continuation of Deferred@996254997 after retry RPC after error@278660832) -> (continuation of Deferred@950805902 after retry RPC after error@1681136498), errback=(continuation of Deferred@656237356 after retry RPC after error@864792416) -> (continuation of Deferred@395808175 after retry RPC after error@498201112) -> (continuation of Deferred@1375336065 after retry RPC after error@1072145265) -> (continuation of Deferred@347519959 after retry RPC after error@101034603) -> (continuation of Deferred@416680404 after retry RPC after error@447274388) -> (continuation of Deferred@996254997 after retry RPC after error@278660832) -> (continuation of Deferred@950805902 after retry RPC after error@1681136498)))
02:59:42.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.501 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.501 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.501 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.501 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.501 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.501 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.521 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.521 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.521 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.521 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.521 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.521 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.521 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.522 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.539 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.540 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.540 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.540 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.540 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.540 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.541 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.561 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.561 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.561 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.561 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.561 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.561 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.562 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.581 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.581 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.581 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.581 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.581 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.581 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.582 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.601 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.601 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.601 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.601 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.602 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.602 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.602 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.602 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.641 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.641 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.641 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.641 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.641 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.641 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.642 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.642 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1271547559(state=PENDING, result=null, callback=(continuation of Deferred@370096385 after retry RPC after error@1434898213) -> (continuation of Deferred@1403099385 after retry RPC after error@1254556500) -> (continuation of Deferred@636484668 after retry RPC after error@1190391875) -> (continuation of Deferred@424332852 after retry RPC after error@1810211818) -> (continuation of Deferred@825339062 after retry RPC after error@1754870587) -> (continuation of Deferred@661486450 after retry RPC after error@1632728710), errback=(continuation of Deferred@370096385 after retry RPC after error@1434898213) -> (continuation of Deferred@1403099385 after retry RPC after error@1254556500) -> (continuation of Deferred@636484668 after retry RPC after error@1190391875) -> (continuation of Deferred@424332852 after retry RPC after error@1810211818) -> (continuation of Deferred@825339062 after retry RPC after error@1754870587) -> (continuation of Deferred@661486450 after retry RPC after error@1632728710)))
02:59:42.642 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1271547559(state=PENDING, result=null, callback=(continuation of Deferred@370096385 after retry RPC after error@1434898213) -> (continuation of Deferred@1403099385 after retry RPC after error@1254556500) -> (continuation of Deferred@636484668 after retry RPC after error@1190391875) -> (continuation of Deferred@424332852 after retry RPC after error@1810211818) -> (continuation of Deferred@825339062 after retry RPC after error@1754870587) -> (continuation of Deferred@661486450 after retry RPC after error@1632728710), errback=(continuation of Deferred@370096385 after retry RPC after error@1434898213) -> (continuation of Deferred@1403099385 after retry RPC after error@1254556500) -> (continuation of Deferred@636484668 after retry RPC after error@1190391875) -> (continuation of Deferred@424332852 after retry RPC after error@1810211818) -> (continuation of Deferred@825339062 after retry RPC after error@1754870587) -> (continuation of Deferred@661486450 after retry RPC after error@1632728710)))
02:59:42.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.661 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.661 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.661 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.661 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.661 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.661 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.661 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.662 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.679 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.680 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.680 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.680 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.681 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.681 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.681 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.700 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.701 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.701 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.701 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.701 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.701 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.721 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.721 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.721 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.721 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.721 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.722 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.721 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.722 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.740 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.741 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.741 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.741 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.741 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.741 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.741 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.781 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.781 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.781 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.781 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.781 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.781 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.781 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.782 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.861 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.861 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.861 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.861 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.861 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.861 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.862 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.862 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1322288622(state=PENDING, result=null, callback=(continuation of Deferred@1960959391 after retry RPC after error@1344600736) -> (continuation of Deferred@1666694414 after retry RPC after error@702724991) -> (continuation of Deferred@1402312441 after retry RPC after error@605263825) -> (continuation of Deferred@2054349200 after retry RPC after error@1182237922) -> (continuation of Deferred@967499915 after retry RPC after error@29316464) -> (continuation of Deferred@1823234742 after retry RPC after error@377914197), errback=(continuation of Deferred@1960959391 after retry RPC after error@1344600736) -> (continuation of Deferred@1666694414 after retry RPC after error@702724991) -> (continuation of Deferred@1402312441 after retry RPC after error@605263825) -> (continuation of Deferred@2054349200 after retry RPC after error@1182237922) -> (continuation of Deferred@967499915 after retry RPC after error@29316464) -> (continuation of Deferred@1823234742 after retry RPC after error@377914197)))
02:59:42.863 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1322288622(state=PENDING, result=null, callback=(continuation of Deferred@1960959391 after retry RPC after error@1344600736) -> (continuation of Deferred@1666694414 after retry RPC after error@702724991) -> (continuation of Deferred@1402312441 after retry RPC after error@605263825) -> (continuation of Deferred@2054349200 after retry RPC after error@1182237922) -> (continuation of Deferred@967499915 after retry RPC after error@29316464) -> (continuation of Deferred@1823234742 after retry RPC after error@377914197), errback=(continuation of Deferred@1960959391 after retry RPC after error@1344600736) -> (continuation of Deferred@1666694414 after retry RPC after error@702724991) -> (continuation of Deferred@1402312441 after retry RPC after error@605263825) -> (continuation of Deferred@2054349200 after retry RPC after error@1182237922) -> (continuation of Deferred@967499915 after retry RPC after error@29316464) -> (continuation of Deferred@1823234742 after retry RPC after error@377914197)))
02:59:42.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.881 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.881 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.881 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.881 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.881 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.881 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.881 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.882 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.899 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.900 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.900 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.901 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.901 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.901 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.901 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.921 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.921 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.921 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.921 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.921 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.921 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.922 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.941 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.941 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.941 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.941 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.941 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.941 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.942 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:42.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:42.961 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:42.961 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.961 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.961 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:42.961 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.961 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.962 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:42.962 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.001 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.001 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.001 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.001 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.001 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.001 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.079 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.080 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.080 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.080 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.081 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.081 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.081 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.081 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.081 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@803237095(state=PENDING, result=null, callback=(continuation of Deferred@1666659738 after retry RPC after error@712989034) -> (continuation of Deferred@1368099277 after retry RPC after error@2107658319) -> (continuation of Deferred@662786436 after retry RPC after error@2003167255) -> (continuation of Deferred@2058207959 after retry RPC after error@91277285) -> (continuation of Deferred@676375486 after retry RPC after error@1882049523) -> (continuation of Deferred@1686246491 after retry RPC after error@1741596865), errback=(continuation of Deferred@1666659738 after retry RPC after error@712989034) -> (continuation of Deferred@1368099277 after retry RPC after error@2107658319) -> (continuation of Deferred@662786436 after retry RPC after error@2003167255) -> (continuation of Deferred@2058207959 after retry RPC after error@91277285) -> (continuation of Deferred@676375486 after retry RPC after error@1882049523) -> (continuation of Deferred@1686246491 after retry RPC after error@1741596865)))
02:59:43.081 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@803237095(state=PENDING, result=null, callback=(continuation of Deferred@1666659738 after retry RPC after error@712989034) -> (continuation of Deferred@1368099277 after retry RPC after error@2107658319) -> (continuation of Deferred@662786436 after retry RPC after error@2003167255) -> (continuation of Deferred@2058207959 after retry RPC after error@91277285) -> (continuation of Deferred@676375486 after retry RPC after error@1882049523) -> (continuation of Deferred@1686246491 after retry RPC after error@1741596865), errback=(continuation of Deferred@1666659738 after retry RPC after error@712989034) -> (continuation of Deferred@1368099277 after retry RPC after error@2107658319) -> (continuation of Deferred@662786436 after retry RPC after error@2003167255) -> (continuation of Deferred@2058207959 after retry RPC after error@91277285) -> (continuation of Deferred@676375486 after retry RPC after error@1882049523) -> (continuation of Deferred@1686246491 after retry RPC after error@1741596865)))
02:59:43.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.100 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.100 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.101 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.101 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.101 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.101 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.121 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.121 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.121 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.121 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.121 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.121 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.141 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.141 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.141 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.141 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.141 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.141 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.142 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.159 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.160 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.160 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.160 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.160 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.161 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.161 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.179 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.180 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.180 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.180 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.180 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.180 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.180 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.199 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.200 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.200 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.200 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.200 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.200 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.201 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.241 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.241 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.241 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.241 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.241 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.241 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.242 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.281 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.281 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.281 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.281 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.282 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.282 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.282 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.282 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.282 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@260699330(state=PENDING, result=null, callback=(continuation of Deferred@1293479113 after retry RPC after error@1547585241) -> (continuation of Deferred@245560178 after retry RPC after error@1313679744) -> (continuation of Deferred@1144650276 after retry RPC after error@1780446452) -> (continuation of Deferred@1941906342 after retry RPC after error@1419318648) -> (continuation of Deferred@1610235157 after retry RPC after error@1322474186) -> (continuation of Deferred@1148973733 after retry RPC after error@1481969031) -> (continuation of Deferred@1456050445 after retry RPC after error@43553812), errback=(continuation of Deferred@1293479113 after retry RPC after error@1547585241) -> (continuation of Deferred@245560178 after retry RPC after error@1313679744) -> (continuation of Deferred@1144650276 after retry RPC after error@1780446452) -> (continuation of Deferred@1941906342 after retry RPC after error@1419318648) -> (continuation of Deferred@1610235157 after retry RPC after error@1322474186) -> (continuation of Deferred@1148973733 after retry RPC after error@1481969031) -> (continuation of Deferred@1456050445 after retry RPC after error@43553812)))
02:59:43.282 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@260699330(state=PENDING, result=null, callback=(continuation of Deferred@1293479113 after retry RPC after error@1547585241) -> (continuation of Deferred@245560178 after retry RPC after error@1313679744) -> (continuation of Deferred@1144650276 after retry RPC after error@1780446452) -> (continuation of Deferred@1941906342 after retry RPC after error@1419318648) -> (continuation of Deferred@1610235157 after retry RPC after error@1322474186) -> (continuation of Deferred@1148973733 after retry RPC after error@1481969031) -> (continuation of Deferred@1456050445 after retry RPC after error@43553812), errback=(continuation of Deferred@1293479113 after retry RPC after error@1547585241) -> (continuation of Deferred@245560178 after retry RPC after error@1313679744) -> (continuation of Deferred@1144650276 after retry RPC after error@1780446452) -> (continuation of Deferred@1941906342 after retry RPC after error@1419318648) -> (continuation of Deferred@1610235157 after retry RPC after error@1322474186) -> (continuation of Deferred@1148973733 after retry RPC after error@1481969031) -> (continuation of Deferred@1456050445 after retry RPC after error@43553812)))
02:59:43.299 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.300 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.300 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.301 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.301 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.301 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.301 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.320 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.321 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.321 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.321 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.321 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.321 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.322 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.341 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.341 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.341 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.341 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.341 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.341 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.361 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.361 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.361 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.361 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.361 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.362 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.362 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.362 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.379 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.381 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.381 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.381 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.381 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.381 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.381 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:43.416157 21179 raft_consensus.cc:534] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 56099a6022da440a81f20e6fe8903fb0)
02:59:43.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:43.416344 21179 raft_consensus.cc:556] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:43.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:43.417068 21179 leader_election.cc:290] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
02:59:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:43.419430 20378 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111): Network error: Client connection negotiation failed: client connection to 127.19.203.254:46111: connect: Connection refused (error 111)
02:59:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:43.419664 20381 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:43.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:43.419828 20381 leader_election.cc:304] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c; no voters: 56099a6022da440a81f20e6fe8903fb0, efb4c69b20bd4f9692d496c33ddbb24e
02:59:43.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:43.420156 21179 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:59:43.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.421 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.421 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.421 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.421 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.421 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.422 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.422 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.481 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.481 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.481 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.481 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.481 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.481 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.481 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.481 [DEBUG - kudu-nio-4] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@879468300(state=PENDING, result=null, callback=(continuation of Deferred@605463036 after retry RPC after error@594192663) -> (continuation of Deferred@1781866724 after retry RPC after error@47071569) -> (continuation of Deferred@331749813 after retry RPC after error@1695688907) -> (continuation of Deferred@652412557 after retry RPC after error@1448546249) -> (continuation of Deferred@1971306247 after retry RPC after error@23041299) -> (continuation of Deferred@1068903643 after retry RPC after error@1366377891), errback=(continuation of Deferred@605463036 after retry RPC after error@594192663) -> (continuation of Deferred@1781866724 after retry RPC after error@47071569) -> (continuation of Deferred@331749813 after retry RPC after error@1695688907) -> (continuation of Deferred@652412557 after retry RPC after error@1448546249) -> (continuation of Deferred@1971306247 after retry RPC after error@23041299) -> (continuation of Deferred@1068903643 after retry RPC after error@1366377891)))
02:59:43.482 [DEBUG - kudu-nio-4] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@879468300(state=PENDING, result=null, callback=(continuation of Deferred@605463036 after retry RPC after error@594192663) -> (continuation of Deferred@1781866724 after retry RPC after error@47071569) -> (continuation of Deferred@331749813 after retry RPC after error@1695688907) -> (continuation of Deferred@652412557 after retry RPC after error@1448546249) -> (continuation of Deferred@1971306247 after retry RPC after error@23041299) -> (continuation of Deferred@1068903643 after retry RPC after error@1366377891), errback=(continuation of Deferred@605463036 after retry RPC after error@594192663) -> (continuation of Deferred@1781866724 after retry RPC after error@47071569) -> (continuation of Deferred@331749813 after retry RPC after error@1695688907) -> (continuation of Deferred@652412557 after retry RPC after error@1448546249) -> (continuation of Deferred@1971306247 after retry RPC after error@23041299) -> (continuation of Deferred@1068903643 after retry RPC after error@1366377891)))
02:59:43.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.501 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.501 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.501 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.501 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.501 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.501 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.521 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.521 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.521 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.521 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.521 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.521 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.522 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.539 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.540 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.540 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.540 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.541 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.541 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.541 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.561 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.561 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.561 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.561 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.561 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.561 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.561 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.581 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.581 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.581 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.581 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.581 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.581 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.601 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.601 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.601 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.601 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.601 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.602 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.602 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.602 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.619 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.620 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.620 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.620 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.621 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.621 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.621 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.701 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.701 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.701 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.701 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.701 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.701 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.701 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.702 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.702 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [203ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@852511162(state=PENDING, result=null, callback=(continuation of Deferred@1494301055 after retry RPC after error@29341593) -> (continuation of Deferred@142360175 after retry RPC after error@953236291) -> (continuation of Deferred@1759193063 after retry RPC after error@337921562) -> (continuation of Deferred@2101513655 after retry RPC after error@1291369087) -> (continuation of Deferred@2100119576 after retry RPC after error@1259324721) -> (continuation of Deferred@583653074 after retry RPC after error@728476598) -> (continuation of Deferred@1007766347 after retry RPC after error@151136181), errback=(continuation of Deferred@1494301055 after retry RPC after error@29341593) -> (continuation of Deferred@142360175 after retry RPC after error@953236291) -> (continuation of Deferred@1759193063 after retry RPC after error@337921562) -> (continuation of Deferred@2101513655 after retry RPC after error@1291369087) -> (continuation of Deferred@2100119576 after retry RPC after error@1259324721) -> (continuation of Deferred@583653074 after retry RPC after error@728476598) -> (continuation of Deferred@1007766347 after retry RPC after error@151136181)))
02:59:43.703 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [203ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@852511162(state=PENDING, result=null, callback=(continuation of Deferred@1494301055 after retry RPC after error@29341593) -> (continuation of Deferred@142360175 after retry RPC after error@953236291) -> (continuation of Deferred@1759193063 after retry RPC after error@337921562) -> (continuation of Deferred@2101513655 after retry RPC after error@1291369087) -> (continuation of Deferred@2100119576 after retry RPC after error@1259324721) -> (continuation of Deferred@583653074 after retry RPC after error@728476598) -> (continuation of Deferred@1007766347 after retry RPC after error@151136181), errback=(continuation of Deferred@1494301055 after retry RPC after error@29341593) -> (continuation of Deferred@142360175 after retry RPC after error@953236291) -> (continuation of Deferred@1759193063 after retry RPC after error@337921562) -> (continuation of Deferred@2101513655 after retry RPC after error@1291369087) -> (continuation of Deferred@2100119576 after retry RPC after error@1259324721) -> (continuation of Deferred@583653074 after retry RPC after error@728476598) -> (continuation of Deferred@1007766347 after retry RPC after error@151136181)))
02:59:43.719 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.720 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.720 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.720 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.720 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.720 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.720 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.721 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.739 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.740 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.740 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.741 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.741 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.741 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.741 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.761 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.761 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.761 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.761 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.761 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.761 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.781 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.781 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.781 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.781 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.781 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.781 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.782 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.801 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.801 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.801 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.801 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.801 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.801 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.802 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.802 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.841 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.841 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.841 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.841 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.841 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.841 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.899 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.900 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.900 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.900 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.900 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.901 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.901 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.901 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1343203604(state=PENDING, result=null, callback=(continuation of Deferred@1253366997 after retry RPC after error@218061883) -> (continuation of Deferred@1863457743 after retry RPC after error@1115183152) -> (continuation of Deferred@784758494 after retry RPC after error@1479473593) -> (continuation of Deferred@656235792 after retry RPC after error@656680987) -> (continuation of Deferred@986307239 after retry RPC after error@352657270) -> (continuation of Deferred@1488790811 after retry RPC after error@1591716583), errback=(continuation of Deferred@1253366997 after retry RPC after error@218061883) -> (continuation of Deferred@1863457743 after retry RPC after error@1115183152) -> (continuation of Deferred@784758494 after retry RPC after error@1479473593) -> (continuation of Deferred@656235792 after retry RPC after error@656680987) -> (continuation of Deferred@986307239 after retry RPC after error@352657270) -> (continuation of Deferred@1488790811 after retry RPC after error@1591716583)))
02:59:43.902 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1343203604(state=PENDING, result=null, callback=(continuation of Deferred@1253366997 after retry RPC after error@218061883) -> (continuation of Deferred@1863457743 after retry RPC after error@1115183152) -> (continuation of Deferred@784758494 after retry RPC after error@1479473593) -> (continuation of Deferred@656235792 after retry RPC after error@656680987) -> (continuation of Deferred@986307239 after retry RPC after error@352657270) -> (continuation of Deferred@1488790811 after retry RPC after error@1591716583), errback=(continuation of Deferred@1253366997 after retry RPC after error@218061883) -> (continuation of Deferred@1863457743 after retry RPC after error@1115183152) -> (continuation of Deferred@784758494 after retry RPC after error@1479473593) -> (continuation of Deferred@656235792 after retry RPC after error@656680987) -> (continuation of Deferred@986307239 after retry RPC after error@352657270) -> (continuation of Deferred@1488790811 after retry RPC after error@1591716583)))
02:59:43.919 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.920 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.920 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.920 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.920 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.920 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.921 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.939 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.940 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.940 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.941 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.941 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.941 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.941 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.961 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.961 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.961 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.961 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.961 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.961 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.962 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:43.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:43.981 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.982 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.982 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:43.982 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.982 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.982 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:43.982 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:43.999 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.000 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.000 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.000 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.000 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.001 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.001 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.021 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.021 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.021 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.021 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.021 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.021 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.079 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.080 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.080 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.080 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.081 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.081 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.081 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.081 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.099 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.100 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.100 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.100 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.100 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.101 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.101 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.120 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.121 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.121 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.121 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.121 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.121 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.122 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@363312144(state=PENDING, result=null, callback=(continuation of Deferred@574305897 after retry RPC after error@130153681) -> (continuation of Deferred@1503566351 after retry RPC after error@1907466776) -> (continuation of Deferred@180509241 after retry RPC after error@49092804) -> (continuation of Deferred@789488756 after retry RPC after error@1397973743) -> (continuation of Deferred@1272510766 after retry RPC after error@555344809) -> (continuation of Deferred@2070863610 after retry RPC after error@862101346) -> (continuation of Deferred@1073680809 after retry RPC after error@1368766804) -> (continuation of Deferred@1551967378 after retry RPC after error@2076125679), errback=(continuation of Deferred@574305897 after retry RPC after error@130153681) -> (continuation of Deferred@1503566351 after retry RPC after error@1907466776) -> (continuation of Deferred@180509241 after retry RPC after error@49092804) -> (continuation of Deferred@789488756 after retry RPC after error@1397973743) -> (continuation of Deferred@1272510766 after retry RPC after error@555344809) -> (continuation of Deferred@2070863610 after retry RPC after error@862101346) -> (continuation of Deferred@1073680809 after retry RPC after error@1368766804) -> (continuation of Deferred@1551967378 after retry RPC after error@2076125679)))
02:59:44.122 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@363312144(state=PENDING, result=null, callback=(continuation of Deferred@574305897 after retry RPC after error@130153681) -> (continuation of Deferred@1503566351 after retry RPC after error@1907466776) -> (continuation of Deferred@180509241 after retry RPC after error@49092804) -> (continuation of Deferred@789488756 after retry RPC after error@1397973743) -> (continuation of Deferred@1272510766 after retry RPC after error@555344809) -> (continuation of Deferred@2070863610 after retry RPC after error@862101346) -> (continuation of Deferred@1073680809 after retry RPC after error@1368766804) -> (continuation of Deferred@1551967378 after retry RPC after error@2076125679), errback=(continuation of Deferred@574305897 after retry RPC after error@130153681) -> (continuation of Deferred@1503566351 after retry RPC after error@1907466776) -> (continuation of Deferred@180509241 after retry RPC after error@49092804) -> (continuation of Deferred@789488756 after retry RPC after error@1397973743) -> (continuation of Deferred@1272510766 after retry RPC after error@555344809) -> (continuation of Deferred@2070863610 after retry RPC after error@862101346) -> (continuation of Deferred@1073680809 after retry RPC after error@1368766804) -> (continuation of Deferred@1551967378 after retry RPC after error@2076125679)))
02:59:44.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.141 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.141 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.141 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.141 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.141 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.141 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.141 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.161 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.161 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.161 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.161 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.161 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.161 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.181 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.181 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.181 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.181 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.181 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.181 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.181 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.182 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.199 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.200 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.200 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.200 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.200 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.200 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.201 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.219 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.220 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.220 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.220 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.220 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.220 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.221 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.259 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.260 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.260 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.260 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.261 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.261 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.261 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.261 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.321 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.321 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.321 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.321 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.321 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.321 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.321 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.322 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.339 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.340 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.340 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.340 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.341 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.341 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.341 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.381 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.381 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.381 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.381 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.381 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.381 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.382 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2066143298(state=PENDING, result=null, callback=(continuation of Deferred@1902180350 after retry RPC after error@785047992) -> (continuation of Deferred@1721601429 after retry RPC after error@733265356) -> (continuation of Deferred@1570594883 after retry RPC after error@614702441) -> (continuation of Deferred@777538893 after retry RPC after error@546099095) -> (continuation of Deferred@371539133 after retry RPC after error@777368517) -> (continuation of Deferred@1116451175 after retry RPC after error@79511254) -> (continuation of Deferred@1002584616 after retry RPC after error@940207374) -> (continuation of Deferred@655746017 after retry RPC after error@1244865115), errback=(continuation of Deferred@1902180350 after retry RPC after error@785047992) -> (continuation of Deferred@1721601429 after retry RPC after error@733265356) -> (continuation of Deferred@1570594883 after retry RPC after error@614702441) -> (continuation of Deferred@777538893 after retry RPC after error@546099095) -> (continuation of Deferred@371539133 after retry RPC after error@777368517) -> (continuation of Deferred@1116451175 after retry RPC after error@79511254) -> (continuation of Deferred@1002584616 after retry RPC after error@940207374) -> (continuation of Deferred@655746017 after retry RPC after error@1244865115)))
02:59:44.382 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2066143298(state=PENDING, result=null, callback=(continuation of Deferred@1902180350 after retry RPC after error@785047992) -> (continuation of Deferred@1721601429 after retry RPC after error@733265356) -> (continuation of Deferred@1570594883 after retry RPC after error@614702441) -> (continuation of Deferred@777538893 after retry RPC after error@546099095) -> (continuation of Deferred@371539133 after retry RPC after error@777368517) -> (continuation of Deferred@1116451175 after retry RPC after error@79511254) -> (continuation of Deferred@1002584616 after retry RPC after error@940207374) -> (continuation of Deferred@655746017 after retry RPC after error@1244865115), errback=(continuation of Deferred@1902180350 after retry RPC after error@785047992) -> (continuation of Deferred@1721601429 after retry RPC after error@733265356) -> (continuation of Deferred@1570594883 after retry RPC after error@614702441) -> (continuation of Deferred@777538893 after retry RPC after error@546099095) -> (continuation of Deferred@371539133 after retry RPC after error@777368517) -> (continuation of Deferred@1116451175 after retry RPC after error@79511254) -> (continuation of Deferred@1002584616 after retry RPC after error@940207374) -> (continuation of Deferred@655746017 after retry RPC after error@1244865115)))
02:59:44.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.401 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.401 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.401 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.401 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.401 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.401 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.401 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.402 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.419 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.420 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.420 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.420 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.420 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.420 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.421 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.421 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.439 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.440 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.440 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.441 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.441 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.441 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.441 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.460 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.460 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.461 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.461 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.461 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.461 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.461 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.481 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.481 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.481 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.481 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.481 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.481 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.482 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.501 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.502 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.502 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.502 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.502 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.502 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.502 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.502 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.561 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.561 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.561 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.561 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.561 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.561 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.561 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.562 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.562 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1891790418(state=PENDING, result=null, callback=(continuation of Deferred@2101805332 after retry RPC after error@892047472) -> (continuation of Deferred@508395922 after retry RPC after error@2107563255) -> (continuation of Deferred@940846577 after retry RPC after error@1372951187) -> (continuation of Deferred@593938474 after retry RPC after error@672448981) -> (continuation of Deferred@838208131 after retry RPC after error@1279936292) -> (continuation of Deferred@1509482783 after retry RPC after error@1475123612), errback=(continuation of Deferred@2101805332 after retry RPC after error@892047472) -> (continuation of Deferred@508395922 after retry RPC after error@2107563255) -> (continuation of Deferred@940846577 after retry RPC after error@1372951187) -> (continuation of Deferred@593938474 after retry RPC after error@672448981) -> (continuation of Deferred@838208131 after retry RPC after error@1279936292) -> (continuation of Deferred@1509482783 after retry RPC after error@1475123612)))
02:59:44.563 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1891790418(state=PENDING, result=null, callback=(continuation of Deferred@2101805332 after retry RPC after error@892047472) -> (continuation of Deferred@508395922 after retry RPC after error@2107563255) -> (continuation of Deferred@940846577 after retry RPC after error@1372951187) -> (continuation of Deferred@593938474 after retry RPC after error@672448981) -> (continuation of Deferred@838208131 after retry RPC after error@1279936292) -> (continuation of Deferred@1509482783 after retry RPC after error@1475123612), errback=(continuation of Deferred@2101805332 after retry RPC after error@892047472) -> (continuation of Deferred@508395922 after retry RPC after error@2107563255) -> (continuation of Deferred@940846577 after retry RPC after error@1372951187) -> (continuation of Deferred@593938474 after retry RPC after error@672448981) -> (continuation of Deferred@838208131 after retry RPC after error@1279936292) -> (continuation of Deferred@1509482783 after retry RPC after error@1475123612)))
02:59:44.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.581 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.581 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.581 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.581 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.581 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.581 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.581 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.582 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.599 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.600 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.600 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.600 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.600 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.600 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.601 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.621 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.621 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.621 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.621 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.621 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.621 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.622 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.641 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.641 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.641 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.641 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.641 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.641 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.641 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.643 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.659 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.660 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.661 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.661 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.661 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.661 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.661 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.662 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.701 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.701 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.701 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.701 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.701 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.701 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.702 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.760 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.760 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.761 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.761 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.761 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.761 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.821 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.821 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.821 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.821 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.821 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.822 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.822 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.823 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.823 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1243668924(state=PENDING, result=null, callback=(continuation of Deferred@321969391 after retry RPC after error@598176876) -> (continuation of Deferred@982770674 after retry RPC after error@2012469277) -> (continuation of Deferred@1655813544 after retry RPC after error@2049812870) -> (continuation of Deferred@1602505026 after retry RPC after error@920539551) -> (continuation of Deferred@1496165913 after retry RPC after error@1322999075) -> (continuation of Deferred@870896611 after retry RPC after error@2013146792) -> (continuation of Deferred@1133976360 after retry RPC after error@1471243324), errback=(continuation of Deferred@321969391 after retry RPC after error@598176876) -> (continuation of Deferred@982770674 after retry RPC after error@2012469277) -> (continuation of Deferred@1655813544 after retry RPC after error@2049812870) -> (continuation of Deferred@1602505026 after retry RPC after error@920539551) -> (continuation of Deferred@1496165913 after retry RPC after error@1322999075) -> (continuation of Deferred@870896611 after retry RPC after error@2013146792) -> (continuation of Deferred@1133976360 after retry RPC after error@1471243324)))
02:59:44.823 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1243668924(state=PENDING, result=null, callback=(continuation of Deferred@321969391 after retry RPC after error@598176876) -> (continuation of Deferred@982770674 after retry RPC after error@2012469277) -> (continuation of Deferred@1655813544 after retry RPC after error@2049812870) -> (continuation of Deferred@1602505026 after retry RPC after error@920539551) -> (continuation of Deferred@1496165913 after retry RPC after error@1322999075) -> (continuation of Deferred@870896611 after retry RPC after error@2013146792) -> (continuation of Deferred@1133976360 after retry RPC after error@1471243324), errback=(continuation of Deferred@321969391 after retry RPC after error@598176876) -> (continuation of Deferred@982770674 after retry RPC after error@2012469277) -> (continuation of Deferred@1655813544 after retry RPC after error@2049812870) -> (continuation of Deferred@1602505026 after retry RPC after error@920539551) -> (continuation of Deferred@1496165913 after retry RPC after error@1322999075) -> (continuation of Deferred@870896611 after retry RPC after error@2013146792) -> (continuation of Deferred@1133976360 after retry RPC after error@1471243324)))
02:59:44.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.840 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.841 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.841 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.841 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.841 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.841 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.861 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.861 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.861 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.861 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.861 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.861 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.881 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.881 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.881 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.881 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.881 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.881 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.882 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.901 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.901 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.901 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.901 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.901 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.901 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.901 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.902 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.919 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.920 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.921 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.921 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.921 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.921 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.921 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.961 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.961 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.961 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.961 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.961 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.961 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:44.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:44.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:44.981 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.981 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.981 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:44.981 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.981 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.981 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:44.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.059 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.060 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.060 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.060 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.060 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.061 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.061 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.079 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.080 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.080 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.080 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.081 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.081 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.081 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.081 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.081 [DEBUG - kudu-nio-6] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2015906130(state=PENDING, result=null, callback=(continuation of Deferred@366575152 after retry RPC after error@316420829) -> (continuation of Deferred@1528543556 after retry RPC after error@1631787436) -> (continuation of Deferred@1068880714 after retry RPC after error@1284261051) -> (continuation of Deferred@1247479287 after retry RPC after error@858585961) -> (continuation of Deferred@580010437 after retry RPC after error@202731696) -> (continuation of Deferred@519288952 after retry RPC after error@1255206780) -> (continuation of Deferred@2063340426 after retry RPC after error@1573612693) -> (continuation of Deferred@1593323308 after retry RPC after error@329154688), errback=(continuation of Deferred@366575152 after retry RPC after error@316420829) -> (continuation of Deferred@1528543556 after retry RPC after error@1631787436) -> (continuation of Deferred@1068880714 after retry RPC after error@1284261051) -> (continuation of Deferred@1247479287 after retry RPC after error@858585961) -> (continuation of Deferred@580010437 after retry RPC after error@202731696) -> (continuation of Deferred@519288952 after retry RPC after error@1255206780) -> (continuation of Deferred@2063340426 after retry RPC after error@1573612693) -> (continuation of Deferred@1593323308 after retry RPC after error@329154688)))
02:59:45.081 [DEBUG - kudu-nio-6] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2015906130(state=PENDING, result=null, callback=(continuation of Deferred@366575152 after retry RPC after error@316420829) -> (continuation of Deferred@1528543556 after retry RPC after error@1631787436) -> (continuation of Deferred@1068880714 after retry RPC after error@1284261051) -> (continuation of Deferred@1247479287 after retry RPC after error@858585961) -> (continuation of Deferred@580010437 after retry RPC after error@202731696) -> (continuation of Deferred@519288952 after retry RPC after error@1255206780) -> (continuation of Deferred@2063340426 after retry RPC after error@1573612693) -> (continuation of Deferred@1593323308 after retry RPC after error@329154688), errback=(continuation of Deferred@366575152 after retry RPC after error@316420829) -> (continuation of Deferred@1528543556 after retry RPC after error@1631787436) -> (continuation of Deferred@1068880714 after retry RPC after error@1284261051) -> (continuation of Deferred@1247479287 after retry RPC after error@858585961) -> (continuation of Deferred@580010437 after retry RPC after error@202731696) -> (continuation of Deferred@519288952 after retry RPC after error@1255206780) -> (continuation of Deferred@2063340426 after retry RPC after error@1573612693) -> (continuation of Deferred@1593323308 after retry RPC after error@329154688)))
02:59:45.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.101 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.101 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.101 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.101 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.101 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.101 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.121 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.121 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.121 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.121 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.121 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.121 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.122 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.141 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.141 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.141 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.141 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.141 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.141 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.141 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.141 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.142 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.142 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.142 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.159 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.160 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.161 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.161 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.161 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.161 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.161 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.181 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.181 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.181 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.181 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.181 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.181 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.221 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.221 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.221 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.221 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.221 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.221 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.221 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.222 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.259 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.260 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.261 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.261 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.261 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.261 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.261 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.261 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.321 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.321 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.321 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.321 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.322 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.322 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.322 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.322 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.322 [DEBUG - kudu-nio-6] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [223ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [223ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [223ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1534004857(state=PENDING, result=null, callback=(continuation of Deferred@983134512 after retry RPC after error@853835586) -> (continuation of Deferred@892015997 after retry RPC after error@378853735) -> (continuation of Deferred@1735735924 after retry RPC after error@1996117943) -> (continuation of Deferred@462890216 after retry RPC after error@729319708) -> (continuation of Deferred@1703981213 after retry RPC after error@1098060919) -> (continuation of Deferred@1675745493 after retry RPC after error@205430236) -> (continuation of Deferred@1825231006 after retry RPC after error@893628892), errback=(continuation of Deferred@983134512 after retry RPC after error@853835586) -> (continuation of Deferred@892015997 after retry RPC after error@378853735) -> (continuation of Deferred@1735735924 after retry RPC after error@1996117943) -> (continuation of Deferred@462890216 after retry RPC after error@729319708) -> (continuation of Deferred@1703981213 after retry RPC after error@1098060919) -> (continuation of Deferred@1675745493 after retry RPC after error@205430236) -> (continuation of Deferred@1825231006 after retry RPC after error@893628892)))
02:59:45.322 [DEBUG - kudu-nio-6] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [223ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [223ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [223ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1534004857(state=PENDING, result=null, callback=(continuation of Deferred@983134512 after retry RPC after error@853835586) -> (continuation of Deferred@892015997 after retry RPC after error@378853735) -> (continuation of Deferred@1735735924 after retry RPC after error@1996117943) -> (continuation of Deferred@462890216 after retry RPC after error@729319708) -> (continuation of Deferred@1703981213 after retry RPC after error@1098060919) -> (continuation of Deferred@1675745493 after retry RPC after error@205430236) -> (continuation of Deferred@1825231006 after retry RPC after error@893628892), errback=(continuation of Deferred@983134512 after retry RPC after error@853835586) -> (continuation of Deferred@892015997 after retry RPC after error@378853735) -> (continuation of Deferred@1735735924 after retry RPC after error@1996117943) -> (continuation of Deferred@462890216 after retry RPC after error@729319708) -> (continuation of Deferred@1703981213 after retry RPC after error@1098060919) -> (continuation of Deferred@1675745493 after retry RPC after error@205430236) -> (continuation of Deferred@1825231006 after retry RPC after error@893628892)))
02:59:45.339 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.340 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.340 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.340 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.340 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.340 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.341 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.360 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.360 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.361 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.361 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.361 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.361 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.381 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.381 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.381 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.381 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.381 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.381 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.401 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.401 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.401 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.401 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.401 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.401 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.402 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.421 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.421 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.421 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.421 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.421 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.421 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.421 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.422 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.439 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.440 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.440 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.440 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.440 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.440 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.441 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.481 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.481 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.481 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.481 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.481 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.481 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.481 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.541 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.541 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.541 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.541 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.541 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.541 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.542 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.542 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.546 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [203ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@259240494(state=PENDING, result=null, callback=(continuation of Deferred@1682765024 after retry RPC after error@733030941) -> (continuation of Deferred@902535504 after retry RPC after error@666611461) -> (continuation of Deferred@609465574 after retry RPC after error@101749683) -> (continuation of Deferred@1878563640 after retry RPC after error@365023813) -> (continuation of Deferred@747417759 after retry RPC after error@771961782) -> (continuation of Deferred@92883989 after retry RPC after error@507639067) -> (continuation of Deferred@1459664388 after retry RPC after error@1621729745), errback=(continuation of Deferred@1682765024 after retry RPC after error@733030941) -> (continuation of Deferred@902535504 after retry RPC after error@666611461) -> (continuation of Deferred@609465574 after retry RPC after error@101749683) -> (continuation of Deferred@1878563640 after retry RPC after error@365023813) -> (continuation of Deferred@747417759 after retry RPC after error@771961782) -> (continuation of Deferred@92883989 after retry RPC after error@507639067) -> (continuation of Deferred@1459664388 after retry RPC after error@1621729745)))
02:59:45.547 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [203ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@259240494(state=PENDING, result=null, callback=(continuation of Deferred@1682765024 after retry RPC after error@733030941) -> (continuation of Deferred@902535504 after retry RPC after error@666611461) -> (continuation of Deferred@609465574 after retry RPC after error@101749683) -> (continuation of Deferred@1878563640 after retry RPC after error@365023813) -> (continuation of Deferred@747417759 after retry RPC after error@771961782) -> (continuation of Deferred@92883989 after retry RPC after error@507639067) -> (continuation of Deferred@1459664388 after retry RPC after error@1621729745), errback=(continuation of Deferred@1682765024 after retry RPC after error@733030941) -> (continuation of Deferred@902535504 after retry RPC after error@666611461) -> (continuation of Deferred@609465574 after retry RPC after error@101749683) -> (continuation of Deferred@1878563640 after retry RPC after error@365023813) -> (continuation of Deferred@747417759 after retry RPC after error@771961782) -> (continuation of Deferred@92883989 after retry RPC after error@507639067) -> (continuation of Deferred@1459664388 after retry RPC after error@1621729745)))
02:59:45.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:45.551534 21194 raft_consensus.cc:534] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 56099a6022da440a81f20e6fe8903fb0)
02:59:45.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:45.551749 21194 raft_consensus.cc:556] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:45.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:45.552659 21194 leader_election.cc:290] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
02:59:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:45.553840 20378 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.203.254:46111: connect: Connection refused (error 111) [suppressed 11 similar messages]
02:59:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:45.555209 20381 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:45.555466 20378 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111): Network error: Client connection negotiation failed: client connection to 127.19.203.254:46111: connect: Connection refused (error 111)
02:59:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:45.555644 20378 leader_election.cc:304] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c; no voters: 56099a6022da440a81f20e6fe8903fb0, efb4c69b20bd4f9692d496c33ddbb24e
02:59:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:45.555963 21194 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:59:45.559 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.560 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.560 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.560 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.560 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.560 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.560 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.561 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.579 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.580 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.580 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.580 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.581 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.581 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.581 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.599 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.600 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.600 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.600 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.601 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.601 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.601 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.620 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.621 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.621 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.621 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.621 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.621 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.640 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.641 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.641 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.641 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.641 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.641 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.681 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.681 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.681 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.681 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.681 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.681 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.681 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.682 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.699 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.700 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.700 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.700 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.700 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.700 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.700 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.701 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@323177946(state=PENDING, result=null, callback=(continuation of Deferred@381572769 after retry RPC after error@1349055720) -> (continuation of Deferred@1876331730 after retry RPC after error@1727148399) -> (continuation of Deferred@35481678 after retry RPC after error@365232977) -> (continuation of Deferred@1320074535 after retry RPC after error@958432468) -> (continuation of Deferred@1343673510 after retry RPC after error@1836358270) -> (continuation of Deferred@1443386738 after retry RPC after error@1806925123), errback=(continuation of Deferred@381572769 after retry RPC after error@1349055720) -> (continuation of Deferred@1876331730 after retry RPC after error@1727148399) -> (continuation of Deferred@35481678 after retry RPC after error@365232977) -> (continuation of Deferred@1320074535 after retry RPC after error@958432468) -> (continuation of Deferred@1343673510 after retry RPC after error@1836358270) -> (continuation of Deferred@1443386738 after retry RPC after error@1806925123)))
02:59:45.701 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@323177946(state=PENDING, result=null, callback=(continuation of Deferred@381572769 after retry RPC after error@1349055720) -> (continuation of Deferred@1876331730 after retry RPC after error@1727148399) -> (continuation of Deferred@35481678 after retry RPC after error@365232977) -> (continuation of Deferred@1320074535 after retry RPC after error@958432468) -> (continuation of Deferred@1343673510 after retry RPC after error@1836358270) -> (continuation of Deferred@1443386738 after retry RPC after error@1806925123), errback=(continuation of Deferred@381572769 after retry RPC after error@1349055720) -> (continuation of Deferred@1876331730 after retry RPC after error@1727148399) -> (continuation of Deferred@35481678 after retry RPC after error@365232977) -> (continuation of Deferred@1320074535 after retry RPC after error@958432468) -> (continuation of Deferred@1343673510 after retry RPC after error@1836358270) -> (continuation of Deferred@1443386738 after retry RPC after error@1806925123)))
02:59:45.719 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.720 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.720 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.720 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.720 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.721 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.721 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.721 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.739 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.740 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.740 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.740 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.740 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.740 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.741 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.761 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.761 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.761 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.761 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.761 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.761 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.762 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.781 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.781 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.781 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.781 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.781 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.781 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.782 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.801 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.801 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.801 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.801 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.801 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.801 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.802 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.802 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.819 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.820 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.820 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.820 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.820 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.820 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.820 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.880 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.881 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.881 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.881 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.881 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.881 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:45.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:45.901 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.901 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.901 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.901 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.901 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:45.901 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:45.902 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:45.980 [DEBUG - pool-40-thread-1] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@689648091(state=PENDING, result=null, callback=(continuation of Deferred@1412133267 after retry RPC after error@882371775) -> (continuation of Deferred@1435131023 after retry RPC after error@636434956) -> (continuation of Deferred@717713611 after retry RPC after error@557744952) -> (continuation of Deferred@651681204 after retry RPC after error@2058665016) -> (continuation of Deferred@1801771661 after retry RPC after error@1041671066) -> (continuation of Deferred@649393041 after retry RPC after error@2034163203) -> (continuation of Deferred@876856091 after retry RPC after error@1061338234) -> (continuation of Deferred@1055647864 after retry RPC after error@824899424), errback=(continuation of Deferred@1412133267 after retry RPC after error@882371775) -> (continuation of Deferred@1435131023 after retry RPC after error@636434956) -> (continuation of Deferred@717713611 after retry RPC after error@557744952) -> (continuation of Deferred@651681204 after retry RPC after error@2058665016) -> (continuation of Deferred@1801771661 after retry RPC after error@1041671066) -> (continuation of Deferred@649393041 after retry RPC after error@2034163203) -> (continuation of Deferred@876856091 after retry RPC after error@1061338234) -> (continuation of Deferred@1055647864 after retry RPC after error@824899424)))
02:59:45.980 [WARN - pool-40-thread-1] (HashedWheelTimer.java:729) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
02:59:45.999 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.000 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.000 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.000 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.000 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.000 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.000 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.019 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.020 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.020 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.020 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.020 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.020 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.020 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.039 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.040 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.040 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.040 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.040 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.041 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.041 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.042 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.059 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.060 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.060 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.060 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.060 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.060 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.060 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.079 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.080 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.080 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.080 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.080 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.081 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.081 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.081 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.121 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.121 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.121 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.121 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.121 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.121 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.179 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.180 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.180 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.180 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.180 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.180 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.180 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.181 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1900082160(state=PENDING, result=null, callback=(continuation of Deferred@87374816 after retry RPC after error@1729545068) -> (continuation of Deferred@2105792191 after retry RPC after error@187331980) -> (continuation of Deferred@661431602 after retry RPC after error@1991253776) -> (continuation of Deferred@367331671 after retry RPC after error@1295751282) -> (continuation of Deferred@264036818 after retry RPC after error@2030184362) -> (continuation of Deferred@1150301804 after retry RPC after error@1442434692), errback=(continuation of Deferred@87374816 after retry RPC after error@1729545068) -> (continuation of Deferred@2105792191 after retry RPC after error@187331980) -> (continuation of Deferred@661431602 after retry RPC after error@1991253776) -> (continuation of Deferred@367331671 after retry RPC after error@1295751282) -> (continuation of Deferred@264036818 after retry RPC after error@2030184362) -> (continuation of Deferred@1150301804 after retry RPC after error@1442434692)))
02:59:46.181 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1900082160(state=PENDING, result=null, callback=(continuation of Deferred@87374816 after retry RPC after error@1729545068) -> (continuation of Deferred@2105792191 after retry RPC after error@187331980) -> (continuation of Deferred@661431602 after retry RPC after error@1991253776) -> (continuation of Deferred@367331671 after retry RPC after error@1295751282) -> (continuation of Deferred@264036818 after retry RPC after error@2030184362) -> (continuation of Deferred@1150301804 after retry RPC after error@1442434692), errback=(continuation of Deferred@87374816 after retry RPC after error@1729545068) -> (continuation of Deferred@2105792191 after retry RPC after error@187331980) -> (continuation of Deferred@661431602 after retry RPC after error@1991253776) -> (continuation of Deferred@367331671 after retry RPC after error@1295751282) -> (continuation of Deferred@264036818 after retry RPC after error@2030184362) -> (continuation of Deferred@1150301804 after retry RPC after error@1442434692)))
02:59:46.199 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.200 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.200 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.200 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.200 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.201 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.201 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.220 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.220 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.220 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.221 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.221 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.221 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.239 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.240 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.240 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.240 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.241 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.241 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.241 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.241 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.260 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.261 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.261 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.261 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.261 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.261 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.261 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.280 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.280 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.281 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.281 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.281 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.281 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.301 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.301 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.301 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.301 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.301 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.301 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.361 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.361 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.361 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.361 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.361 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.361 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.361 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.362 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.362 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [163ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@775819042(state=PENDING, result=null, callback=(continuation of Deferred@1501194564 after retry RPC after error@1481790609) -> (continuation of Deferred@1901020600 after retry RPC after error@794170284) -> (continuation of Deferred@627863371 after retry RPC after error@1867530667) -> (continuation of Deferred@1681505148 after retry RPC after error@2070748964) -> (continuation of Deferred@1027420469 after retry RPC after error@1343058464) -> (continuation of Deferred@2043068678 after retry RPC after error@1782670629), errback=(continuation of Deferred@1501194564 after retry RPC after error@1481790609) -> (continuation of Deferred@1901020600 after retry RPC after error@794170284) -> (continuation of Deferred@627863371 after retry RPC after error@1867530667) -> (continuation of Deferred@1681505148 after retry RPC after error@2070748964) -> (continuation of Deferred@1027420469 after retry RPC after error@1343058464) -> (continuation of Deferred@2043068678 after retry RPC after error@1782670629)))
02:59:46.362 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [163ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@775819042(state=PENDING, result=null, callback=(continuation of Deferred@1501194564 after retry RPC after error@1481790609) -> (continuation of Deferred@1901020600 after retry RPC after error@794170284) -> (continuation of Deferred@627863371 after retry RPC after error@1867530667) -> (continuation of Deferred@1681505148 after retry RPC after error@2070748964) -> (continuation of Deferred@1027420469 after retry RPC after error@1343058464) -> (continuation of Deferred@2043068678 after retry RPC after error@1782670629), errback=(continuation of Deferred@1501194564 after retry RPC after error@1481790609) -> (continuation of Deferred@1901020600 after retry RPC after error@794170284) -> (continuation of Deferred@627863371 after retry RPC after error@1867530667) -> (continuation of Deferred@1681505148 after retry RPC after error@2070748964) -> (continuation of Deferred@1027420469 after retry RPC after error@1343058464) -> (continuation of Deferred@2043068678 after retry RPC after error@1782670629)))
02:59:46.379 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.380 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.381 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.381 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.381 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.381 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.381 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.401 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.401 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.401 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.401 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.401 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.401 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.421 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.421 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.421 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.421 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.421 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.421 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.421 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.441 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.441 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.441 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.441 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.441 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.441 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.442 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.461 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.461 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.461 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.461 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.461 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.461 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.462 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.481 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.481 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.481 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.481 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.481 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.481 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.481 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.482 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.519 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.520 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.520 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.520 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.521 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.521 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.521 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.561 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.561 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.561 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.561 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.561 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.561 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.561 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.562 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@794925936(state=PENDING, result=null, callback=(continuation of Deferred@1493697183 after retry RPC after error@127996393) -> (continuation of Deferred@929178264 after retry RPC after error@343639327) -> (continuation of Deferred@226390517 after retry RPC after error@48175082) -> (continuation of Deferred@613385163 after retry RPC after error@1821681976) -> (continuation of Deferred@303795334 after retry RPC after error@1753327976) -> (continuation of Deferred@810090088 after retry RPC after error@2036096048) -> (continuation of Deferred@938315809 after retry RPC after error@1388488029), errback=(continuation of Deferred@1493697183 after retry RPC after error@127996393) -> (continuation of Deferred@929178264 after retry RPC after error@343639327) -> (continuation of Deferred@226390517 after retry RPC after error@48175082) -> (continuation of Deferred@613385163 after retry RPC after error@1821681976) -> (continuation of Deferred@303795334 after retry RPC after error@1753327976) -> (continuation of Deferred@810090088 after retry RPC after error@2036096048) -> (continuation of Deferred@938315809 after retry RPC after error@1388488029)))
02:59:46.562 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@794925936(state=PENDING, result=null, callback=(continuation of Deferred@1493697183 after retry RPC after error@127996393) -> (continuation of Deferred@929178264 after retry RPC after error@343639327) -> (continuation of Deferred@226390517 after retry RPC after error@48175082) -> (continuation of Deferred@613385163 after retry RPC after error@1821681976) -> (continuation of Deferred@303795334 after retry RPC after error@1753327976) -> (continuation of Deferred@810090088 after retry RPC after error@2036096048) -> (continuation of Deferred@938315809 after retry RPC after error@1388488029), errback=(continuation of Deferred@1493697183 after retry RPC after error@127996393) -> (continuation of Deferred@929178264 after retry RPC after error@343639327) -> (continuation of Deferred@226390517 after retry RPC after error@48175082) -> (continuation of Deferred@613385163 after retry RPC after error@1821681976) -> (continuation of Deferred@303795334 after retry RPC after error@1753327976) -> (continuation of Deferred@810090088 after retry RPC after error@2036096048) -> (continuation of Deferred@938315809 after retry RPC after error@1388488029)))
02:59:46.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.581 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.581 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.581 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.581 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.581 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.581 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.581 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.601 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.601 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.601 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.601 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.601 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.601 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.602 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.602 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.620 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.621 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.621 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.621 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.621 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.621 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.640 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.640 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.641 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.641 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.641 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.641 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.661 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.661 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.661 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.661 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.661 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.661 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.662 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.701 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.701 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.701 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.701 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.701 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.701 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.701 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.702 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.740 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.740 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.741 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.741 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.741 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.741 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.819 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.820 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.820 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.820 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.820 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.820 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.820 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.821 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1924688870(state=PENDING, result=null, callback=(continuation of Deferred@177512080 after retry RPC after error@384883484) -> (continuation of Deferred@818732686 after retry RPC after error@26895402) -> (continuation of Deferred@794036887 after retry RPC after error@1072438362) -> (continuation of Deferred@1770167687 after retry RPC after error@996872105) -> (continuation of Deferred@887401185 after retry RPC after error@1798663915) -> (continuation of Deferred@380025007 after retry RPC after error@1968434069) -> (continuation of Deferred@1693574268 after retry RPC after error@619940927), errback=(continuation of Deferred@177512080 after retry RPC after error@384883484) -> (continuation of Deferred@818732686 after retry RPC after error@26895402) -> (continuation of Deferred@794036887 after retry RPC after error@1072438362) -> (continuation of Deferred@1770167687 after retry RPC after error@996872105) -> (continuation of Deferred@887401185 after retry RPC after error@1798663915) -> (continuation of Deferred@380025007 after retry RPC after error@1968434069) -> (continuation of Deferred@1693574268 after retry RPC after error@619940927)))
02:59:46.822 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1924688870(state=PENDING, result=null, callback=(continuation of Deferred@177512080 after retry RPC after error@384883484) -> (continuation of Deferred@818732686 after retry RPC after error@26895402) -> (continuation of Deferred@794036887 after retry RPC after error@1072438362) -> (continuation of Deferred@1770167687 after retry RPC after error@996872105) -> (continuation of Deferred@887401185 after retry RPC after error@1798663915) -> (continuation of Deferred@380025007 after retry RPC after error@1968434069) -> (continuation of Deferred@1693574268 after retry RPC after error@619940927), errback=(continuation of Deferred@177512080 after retry RPC after error@384883484) -> (continuation of Deferred@818732686 after retry RPC after error@26895402) -> (continuation of Deferred@794036887 after retry RPC after error@1072438362) -> (continuation of Deferred@1770167687 after retry RPC after error@996872105) -> (continuation of Deferred@887401185 after retry RPC after error@1798663915) -> (continuation of Deferred@380025007 after retry RPC after error@1968434069) -> (continuation of Deferred@1693574268 after retry RPC after error@619940927)))
02:59:46.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.840 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.840 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.841 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.841 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.841 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.841 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.842 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.861 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.861 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.861 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.861 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.861 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.861 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.862 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.881 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.881 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.881 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.881 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.881 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.881 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.882 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.901 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.901 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.901 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.901 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.901 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.901 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.901 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.902 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.920 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.920 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.921 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.921 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.921 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.921 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.941 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.941 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.941 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.941 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.941 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.941 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:46.981 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:46.981 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.981 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.981 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:46.981 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.982 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.982 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:46.982 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:46.982 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=143), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [3ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@107981868(state=PENDING, result=null, callback=(continuation of Deferred@1053121194 after retry RPC after error@548826103) -> (continuation of Deferred@2114027800 after retry RPC after error@99900472) -> (continuation of Deferred@292570662 after retry RPC after error@42626604) -> (continuation of Deferred@1886467620 after retry RPC after error@1472442766) -> (continuation of Deferred@1150393382 after retry RPC after error@364605846) -> (continuation of Deferred@1820501267 after retry RPC after error@638006988), errback=(continuation of Deferred@1053121194 after retry RPC after error@548826103) -> (continuation of Deferred@2114027800 after retry RPC after error@99900472) -> (continuation of Deferred@292570662 after retry RPC after error@42626604) -> (continuation of Deferred@1886467620 after retry RPC after error@1472442766) -> (continuation of Deferred@1150393382 after retry RPC after error@364605846) -> (continuation of Deferred@1820501267 after retry RPC after error@638006988)))
02:59:46.983 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=143), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [3ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@107981868(state=PENDING, result=null, callback=(continuation of Deferred@1053121194 after retry RPC after error@548826103) -> (continuation of Deferred@2114027800 after retry RPC after error@99900472) -> (continuation of Deferred@292570662 after retry RPC after error@42626604) -> (continuation of Deferred@1886467620 after retry RPC after error@1472442766) -> (continuation of Deferred@1150393382 after retry RPC after error@364605846) -> (continuation of Deferred@1820501267 after retry RPC after error@638006988), errback=(continuation of Deferred@1053121194 after retry RPC after error@548826103) -> (continuation of Deferred@2114027800 after retry RPC after error@99900472) -> (continuation of Deferred@292570662 after retry RPC after error@42626604) -> (continuation of Deferred@1886467620 after retry RPC after error@1472442766) -> (continuation of Deferred@1150393382 after retry RPC after error@364605846) -> (continuation of Deferred@1820501267 after retry RPC after error@638006988)))
02:59:47.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.000 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.001 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.001 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.001 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.001 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.001 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.021 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.021 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.021 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.021 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.021 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.021 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.041 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.041 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.041 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.041 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.041 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.041 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.041 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.061 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.061 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.061 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.061 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.061 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.061 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.081 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.081 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.081 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.081 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.081 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.082 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.082 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.101 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.101 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.101 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.101 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.101 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.101 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.101 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.101 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.102 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.102 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.141 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.141 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.141 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.141 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.141 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.141 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.142 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.142 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.181 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.181 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.181 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.181 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.181 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.181 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.182 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1474422551(state=PENDING, result=null, callback=(continuation of Deferred@1193227864 after retry RPC after error@852397069) -> (continuation of Deferred@800870374 after retry RPC after error@2008927321) -> (continuation of Deferred@1236175104 after retry RPC after error@264686300) -> (continuation of Deferred@1120430013 after retry RPC after error@1527690425) -> (continuation of Deferred@1725288529 after retry RPC after error@1610376829) -> (continuation of Deferred@1577584560 after retry RPC after error@1888340916) -> (continuation of Deferred@940759724 after retry RPC after error@81894493), errback=(continuation of Deferred@1193227864 after retry RPC after error@852397069) -> (continuation of Deferred@800870374 after retry RPC after error@2008927321) -> (continuation of Deferred@1236175104 after retry RPC after error@264686300) -> (continuation of Deferred@1120430013 after retry RPC after error@1527690425) -> (continuation of Deferred@1725288529 after retry RPC after error@1610376829) -> (continuation of Deferred@1577584560 after retry RPC after error@1888340916) -> (continuation of Deferred@940759724 after retry RPC after error@81894493)))
02:59:47.182 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1474422551(state=PENDING, result=null, callback=(continuation of Deferred@1193227864 after retry RPC after error@852397069) -> (continuation of Deferred@800870374 after retry RPC after error@2008927321) -> (continuation of Deferred@1236175104 after retry RPC after error@264686300) -> (continuation of Deferred@1120430013 after retry RPC after error@1527690425) -> (continuation of Deferred@1725288529 after retry RPC after error@1610376829) -> (continuation of Deferred@1577584560 after retry RPC after error@1888340916) -> (continuation of Deferred@940759724 after retry RPC after error@81894493), errback=(continuation of Deferred@1193227864 after retry RPC after error@852397069) -> (continuation of Deferred@800870374 after retry RPC after error@2008927321) -> (continuation of Deferred@1236175104 after retry RPC after error@264686300) -> (continuation of Deferred@1120430013 after retry RPC after error@1527690425) -> (continuation of Deferred@1725288529 after retry RPC after error@1610376829) -> (continuation of Deferred@1577584560 after retry RPC after error@1888340916) -> (continuation of Deferred@940759724 after retry RPC after error@81894493)))
02:59:47.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.201 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.201 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.201 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.201 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.201 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.201 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.202 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.221 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.221 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.221 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.221 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.221 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.221 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.222 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.241 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.241 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.241 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.241 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.241 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.241 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.241 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.242 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.261 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.261 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.261 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.261 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.261 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.262 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.262 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.262 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.279 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.280 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.280 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.281 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.281 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.281 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.281 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.301 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.301 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.301 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.301 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.301 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.301 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.361 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.361 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.361 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.361 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.361 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.361 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.362 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.362 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.441 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.441 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.441 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.441 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.441 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.441 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.442 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1233621602(state=PENDING, result=null, callback=(continuation of Deferred@217080559 after retry RPC after error@449630442) -> (continuation of Deferred@1925294235 after retry RPC after error@127934499) -> (continuation of Deferred@138988834 after retry RPC after error@1954486081) -> (continuation of Deferred@184941605 after retry RPC after error@220357726) -> (continuation of Deferred@1137543429 after retry RPC after error@1325685538) -> (continuation of Deferred@1808032307 after retry RPC after error@1968896897) -> (continuation of Deferred@1525727630 after retry RPC after error@926960825), errback=(continuation of Deferred@217080559 after retry RPC after error@449630442) -> (continuation of Deferred@1925294235 after retry RPC after error@127934499) -> (continuation of Deferred@138988834 after retry RPC after error@1954486081) -> (continuation of Deferred@184941605 after retry RPC after error@220357726) -> (continuation of Deferred@1137543429 after retry RPC after error@1325685538) -> (continuation of Deferred@1808032307 after retry RPC after error@1968896897) -> (continuation of Deferred@1525727630 after retry RPC after error@926960825)))
02:59:47.442 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1233621602(state=PENDING, result=null, callback=(continuation of Deferred@217080559 after retry RPC after error@449630442) -> (continuation of Deferred@1925294235 after retry RPC after error@127934499) -> (continuation of Deferred@138988834 after retry RPC after error@1954486081) -> (continuation of Deferred@184941605 after retry RPC after error@220357726) -> (continuation of Deferred@1137543429 after retry RPC after error@1325685538) -> (continuation of Deferred@1808032307 after retry RPC after error@1968896897) -> (continuation of Deferred@1525727630 after retry RPC after error@926960825), errback=(continuation of Deferred@217080559 after retry RPC after error@449630442) -> (continuation of Deferred@1925294235 after retry RPC after error@127934499) -> (continuation of Deferred@138988834 after retry RPC after error@1954486081) -> (continuation of Deferred@184941605 after retry RPC after error@220357726) -> (continuation of Deferred@1137543429 after retry RPC after error@1325685538) -> (continuation of Deferred@1808032307 after retry RPC after error@1968896897) -> (continuation of Deferred@1525727630 after retry RPC after error@926960825)))
02:59:47.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.461 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.461 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.461 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.461 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.461 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.461 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.461 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.481 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.481 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.481 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.481 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.481 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.481 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.481 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.500 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.500 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.501 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.501 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.501 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.501 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.521 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.521 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.521 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.521 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.521 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.521 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.540 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.540 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.541 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.541 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.541 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.541 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.581 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.581 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.581 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.581 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.581 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.581 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.601 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.601 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.601 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.601 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.601 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.601 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.602 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.720 [DEBUG - pool-40-thread-1] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1619127797(state=PENDING, result=null, callback=(continuation of Deferred@1946936259 after retry RPC after error@990292057) -> (continuation of Deferred@2124912125 after retry RPC after error@296100249) -> (continuation of Deferred@983092629 after retry RPC after error@886238975) -> (continuation of Deferred@1981324773 after retry RPC after error@1452458070) -> (continuation of Deferred@714754932 after retry RPC after error@161283534) -> (continuation of Deferred@995733165 after retry RPC after error@2072599008) -> (continuation of Deferred@61893646 after retry RPC after error@1840026427), errback=(continuation of Deferred@1946936259 after retry RPC after error@990292057) -> (continuation of Deferred@2124912125 after retry RPC after error@296100249) -> (continuation of Deferred@983092629 after retry RPC after error@886238975) -> (continuation of Deferred@1981324773 after retry RPC after error@1452458070) -> (continuation of Deferred@714754932 after retry RPC after error@161283534) -> (continuation of Deferred@995733165 after retry RPC after error@2072599008) -> (continuation of Deferred@61893646 after retry RPC after error@1840026427)))
02:59:47.721 [WARN - pool-40-thread-1] (HashedWheelTimer.java:729) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
02:59:47.739 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.740 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.740 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.740 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.740 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.740 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.740 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.759 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.760 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.760 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.760 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.761 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.761 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.761 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.781 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.781 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.781 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.781 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.781 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.781 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.801 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.801 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.801 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.801 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.801 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.801 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.802 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.821 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.821 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.821 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.821 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.822 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.822 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.822 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.822 [WARN - kudu-nio-5] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.839 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.840 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.840 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.840 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.840 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.841 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.841 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.900 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.901 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.901 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.901 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.901 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.901 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.920 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.920 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.921 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.921 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.921 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.921 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.922 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1143760879(state=PENDING, result=null, callback=(continuation of Deferred@807779042 after retry RPC after error@2044485235) -> (continuation of Deferred@2030979916 after retry RPC after error@1580877981) -> (continuation of Deferred@1129650962 after retry RPC after error@1482812109) -> (continuation of Deferred@1396608678 after retry RPC after error@1607224060) -> (continuation of Deferred@875186076 after retry RPC after error@1674245206) -> (continuation of Deferred@1151737143 after retry RPC after error@2030056693) -> (continuation of Deferred@527473390 after retry RPC after error@732734738), errback=(continuation of Deferred@807779042 after retry RPC after error@2044485235) -> (continuation of Deferred@2030979916 after retry RPC after error@1580877981) -> (continuation of Deferred@1129650962 after retry RPC after error@1482812109) -> (continuation of Deferred@1396608678 after retry RPC after error@1607224060) -> (continuation of Deferred@875186076 after retry RPC after error@1674245206) -> (continuation of Deferred@1151737143 after retry RPC after error@2030056693) -> (continuation of Deferred@527473390 after retry RPC after error@732734738)))
02:59:47.922 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1143760879(state=PENDING, result=null, callback=(continuation of Deferred@807779042 after retry RPC after error@2044485235) -> (continuation of Deferred@2030979916 after retry RPC after error@1580877981) -> (continuation of Deferred@1129650962 after retry RPC after error@1482812109) -> (continuation of Deferred@1396608678 after retry RPC after error@1607224060) -> (continuation of Deferred@875186076 after retry RPC after error@1674245206) -> (continuation of Deferred@1151737143 after retry RPC after error@2030056693) -> (continuation of Deferred@527473390 after retry RPC after error@732734738), errback=(continuation of Deferred@807779042 after retry RPC after error@2044485235) -> (continuation of Deferred@2030979916 after retry RPC after error@1580877981) -> (continuation of Deferred@1129650962 after retry RPC after error@1482812109) -> (continuation of Deferred@1396608678 after retry RPC after error@1607224060) -> (continuation of Deferred@875186076 after retry RPC after error@1674245206) -> (continuation of Deferred@1151737143 after retry RPC after error@2030056693) -> (continuation of Deferred@527473390 after retry RPC after error@732734738)))
02:59:47.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.940 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.940 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.940 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.941 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.941 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.941 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.961 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.961 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.961 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.961 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.961 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.961 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:47.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:47.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:47.980 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.980 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.981 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:47.981 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.981 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.981 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:47.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.001 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.001 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.001 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.001 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.001 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.001 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.021 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.021 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.021 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.021 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.021 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.021 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.022 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.061 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.061 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.061 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.061 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.061 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.061 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.061 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.062 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.099 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.100 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.100 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.100 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.100 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.100 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.101 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:48.105764 21217 raft_consensus.cc:534] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 56099a6022da440a81f20e6fe8903fb0)
02:59:48.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:48.105968 21217 raft_consensus.cc:556] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:48.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:48.106611 21217 leader_election.cc:290] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
02:59:48.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:48.109027 20381 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:48.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:48.109227 20378 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111): Network error: Client connection negotiation failed: client connection to 127.19.203.254:46111: connect: Connection refused (error 111)
02:59:48.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:48.109334 20378 leader_election.cc:304] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c; no voters: 56099a6022da440a81f20e6fe8903fb0, efb4c69b20bd4f9692d496c33ddbb24e
02:59:48.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:48.109599 21217 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:59:48.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.120 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.120 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.121 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.121 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.121 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.121 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.122 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1935981181(state=PENDING, result=null, callback=(continuation of Deferred@1846667065 after retry RPC after error@1744873492) -> (continuation of Deferred@639540359 after retry RPC after error@1895787883) -> (continuation of Deferred@297170018 after retry RPC after error@2090544712) -> (continuation of Deferred@775122936 after retry RPC after error@632085344) -> (continuation of Deferred@127647443 after retry RPC after error@576137478) -> (continuation of Deferred@388947483 after retry RPC after error@1419403333) -> (continuation of Deferred@246227235 after retry RPC after error@363407081), errback=(continuation of Deferred@1846667065 after retry RPC after error@1744873492) -> (continuation of Deferred@639540359 after retry RPC after error@1895787883) -> (continuation of Deferred@297170018 after retry RPC after error@2090544712) -> (continuation of Deferred@775122936 after retry RPC after error@632085344) -> (continuation of Deferred@127647443 after retry RPC after error@576137478) -> (continuation of Deferred@388947483 after retry RPC after error@1419403333) -> (continuation of Deferred@246227235 after retry RPC after error@363407081)))
02:59:48.122 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1935981181(state=PENDING, result=null, callback=(continuation of Deferred@1846667065 after retry RPC after error@1744873492) -> (continuation of Deferred@639540359 after retry RPC after error@1895787883) -> (continuation of Deferred@297170018 after retry RPC after error@2090544712) -> (continuation of Deferred@775122936 after retry RPC after error@632085344) -> (continuation of Deferred@127647443 after retry RPC after error@576137478) -> (continuation of Deferred@388947483 after retry RPC after error@1419403333) -> (continuation of Deferred@246227235 after retry RPC after error@363407081), errback=(continuation of Deferred@1846667065 after retry RPC after error@1744873492) -> (continuation of Deferred@639540359 after retry RPC after error@1895787883) -> (continuation of Deferred@297170018 after retry RPC after error@2090544712) -> (continuation of Deferred@775122936 after retry RPC after error@632085344) -> (continuation of Deferred@127647443 after retry RPC after error@576137478) -> (continuation of Deferred@388947483 after retry RPC after error@1419403333) -> (continuation of Deferred@246227235 after retry RPC after error@363407081)))
02:59:48.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.141 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.141 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.141 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.141 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.141 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.141 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.142 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.161 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.161 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.161 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.161 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.161 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.161 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.161 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.162 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.179 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.181 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.181 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.181 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.181 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.181 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.181 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.201 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.201 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.201 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.201 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.201 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.201 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.201 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.202 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.219 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.220 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.220 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.221 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.221 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.221 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.221 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.241 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.241 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.241 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.241 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.241 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.241 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.241 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.261 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.261 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.261 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.261 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.261 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.261 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.262 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.281 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.281 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.281 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.281 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.281 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.281 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.281 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.281 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.282 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.282 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.282 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@197122591(state=PENDING, result=null, callback=(continuation of Deferred@832177780 after retry RPC after error@1889292027) -> (continuation of Deferred@2055462616 after retry RPC after error@253336757) -> (continuation of Deferred@1850835588 after retry RPC after error@2083058211) -> (continuation of Deferred@406423136 after retry RPC after error@1535532518) -> (continuation of Deferred@958505392 after retry RPC after error@35911005) -> (continuation of Deferred@1451059711 after retry RPC after error@1073575867) -> (continuation of Deferred@199589333 after retry RPC after error@384952553), errback=(continuation of Deferred@832177780 after retry RPC after error@1889292027) -> (continuation of Deferred@2055462616 after retry RPC after error@253336757) -> (continuation of Deferred@1850835588 after retry RPC after error@2083058211) -> (continuation of Deferred@406423136 after retry RPC after error@1535532518) -> (continuation of Deferred@958505392 after retry RPC after error@35911005) -> (continuation of Deferred@1451059711 after retry RPC after error@1073575867) -> (continuation of Deferred@199589333 after retry RPC after error@384952553)))
02:59:48.282 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@197122591(state=PENDING, result=null, callback=(continuation of Deferred@832177780 after retry RPC after error@1889292027) -> (continuation of Deferred@2055462616 after retry RPC after error@253336757) -> (continuation of Deferred@1850835588 after retry RPC after error@2083058211) -> (continuation of Deferred@406423136 after retry RPC after error@1535532518) -> (continuation of Deferred@958505392 after retry RPC after error@35911005) -> (continuation of Deferred@1451059711 after retry RPC after error@1073575867) -> (continuation of Deferred@199589333 after retry RPC after error@384952553), errback=(continuation of Deferred@832177780 after retry RPC after error@1889292027) -> (continuation of Deferred@2055462616 after retry RPC after error@253336757) -> (continuation of Deferred@1850835588 after retry RPC after error@2083058211) -> (continuation of Deferred@406423136 after retry RPC after error@1535532518) -> (continuation of Deferred@958505392 after retry RPC after error@35911005) -> (continuation of Deferred@1451059711 after retry RPC after error@1073575867) -> (continuation of Deferred@199589333 after retry RPC after error@384952553)))
02:59:48.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.301 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.301 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.301 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.301 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.301 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.301 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.302 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.321 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.321 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.321 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.321 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.321 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.321 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.322 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.341 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.341 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.341 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.341 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.341 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.341 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.341 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.342 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.361 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.361 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.361 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.361 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.361 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.361 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.361 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.362 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.379 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.380 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.380 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.380 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.380 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.380 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.380 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.419 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.420 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.420 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.420 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.421 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.421 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.421 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.421 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.461 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.461 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.461 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.461 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.461 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.462 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.462 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.481 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.481 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.481 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.482 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.482 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.482 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.482 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.541 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.541 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.541 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.541 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.541 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.541 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.542 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.542 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1633278580(state=PENDING, result=null, callback=(continuation of Deferred@1425661118 after retry RPC after error@1432935632) -> (continuation of Deferred@878662070 after retry RPC after error@461027123) -> (continuation of Deferred@33326286 after retry RPC after error@308364925) -> (continuation of Deferred@1560110950 after retry RPC after error@317755621) -> (continuation of Deferred@2001714305 after retry RPC after error@1871007419) -> (continuation of Deferred@1484046970 after retry RPC after error@1318393385) -> (continuation of Deferred@1144258766 after retry RPC after error@16871013) -> (continuation of Deferred@2132282157 after retry RPC after error@433745593), errback=(continuation of Deferred@1425661118 after retry RPC after error@1432935632) -> (continuation of Deferred@878662070 after retry RPC after error@461027123) -> (continuation of Deferred@33326286 after retry RPC after error@308364925) -> (continuation of Deferred@1560110950 after retry RPC after error@317755621) -> (continuation of Deferred@2001714305 after retry RPC after error@1871007419) -> (continuation of Deferred@1484046970 after retry RPC after error@1318393385) -> (continuation of Deferred@1144258766 after retry RPC after error@16871013) -> (continuation of Deferred@2132282157 after retry RPC after error@433745593)))
02:59:48.542 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1633278580(state=PENDING, result=null, callback=(continuation of Deferred@1425661118 after retry RPC after error@1432935632) -> (continuation of Deferred@878662070 after retry RPC after error@461027123) -> (continuation of Deferred@33326286 after retry RPC after error@308364925) -> (continuation of Deferred@1560110950 after retry RPC after error@317755621) -> (continuation of Deferred@2001714305 after retry RPC after error@1871007419) -> (continuation of Deferred@1484046970 after retry RPC after error@1318393385) -> (continuation of Deferred@1144258766 after retry RPC after error@16871013) -> (continuation of Deferred@2132282157 after retry RPC after error@433745593), errback=(continuation of Deferred@1425661118 after retry RPC after error@1432935632) -> (continuation of Deferred@878662070 after retry RPC after error@461027123) -> (continuation of Deferred@33326286 after retry RPC after error@308364925) -> (continuation of Deferred@1560110950 after retry RPC after error@317755621) -> (continuation of Deferred@2001714305 after retry RPC after error@1871007419) -> (continuation of Deferred@1484046970 after retry RPC after error@1318393385) -> (continuation of Deferred@1144258766 after retry RPC after error@16871013) -> (continuation of Deferred@2132282157 after retry RPC after error@433745593)))
02:59:48.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.561 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.561 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.561 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.561 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.562 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.562 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.562 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.562 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.581 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.581 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.581 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.581 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.581 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.581 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.582 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.601 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.601 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.601 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.601 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.601 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.601 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.621 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.621 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.621 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.621 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.621 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.621 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.622 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.641 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.641 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.641 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.641 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.641 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.641 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.642 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.661 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.661 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.661 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.661 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.661 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.661 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.662 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.699 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.700 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.700 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.700 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.700 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.701 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.701 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.779 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.780 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.780 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.781 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.781 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.781 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.781 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.782 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1224778788(state=PENDING, result=null, callback=(continuation of Deferred@1356017490 after retry RPC after error@730313312) -> (continuation of Deferred@29251148 after retry RPC after error@1297347547) -> (continuation of Deferred@78904476 after retry RPC after error@313608920) -> (continuation of Deferred@1632527092 after retry RPC after error@869369901) -> (continuation of Deferred@681428439 after retry RPC after error@811773334) -> (continuation of Deferred@573124780 after retry RPC after error@486949924) -> (continuation of Deferred@1534302510 after retry RPC after error@2119729611), errback=(continuation of Deferred@1356017490 after retry RPC after error@730313312) -> (continuation of Deferred@29251148 after retry RPC after error@1297347547) -> (continuation of Deferred@78904476 after retry RPC after error@313608920) -> (continuation of Deferred@1632527092 after retry RPC after error@869369901) -> (continuation of Deferred@681428439 after retry RPC after error@811773334) -> (continuation of Deferred@573124780 after retry RPC after error@486949924) -> (continuation of Deferred@1534302510 after retry RPC after error@2119729611)))
02:59:48.782 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1224778788(state=PENDING, result=null, callback=(continuation of Deferred@1356017490 after retry RPC after error@730313312) -> (continuation of Deferred@29251148 after retry RPC after error@1297347547) -> (continuation of Deferred@78904476 after retry RPC after error@313608920) -> (continuation of Deferred@1632527092 after retry RPC after error@869369901) -> (continuation of Deferred@681428439 after retry RPC after error@811773334) -> (continuation of Deferred@573124780 after retry RPC after error@486949924) -> (continuation of Deferred@1534302510 after retry RPC after error@2119729611), errback=(continuation of Deferred@1356017490 after retry RPC after error@730313312) -> (continuation of Deferred@29251148 after retry RPC after error@1297347547) -> (continuation of Deferred@78904476 after retry RPC after error@313608920) -> (continuation of Deferred@1632527092 after retry RPC after error@869369901) -> (continuation of Deferred@681428439 after retry RPC after error@811773334) -> (continuation of Deferred@573124780 after retry RPC after error@486949924) -> (continuation of Deferred@1534302510 after retry RPC after error@2119729611)))
02:59:48.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.800 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.800 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.801 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.801 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.801 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.801 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.820 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.821 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.821 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.821 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.821 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.821 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.840 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.840 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.841 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.841 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.841 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.841 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.860 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.861 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.861 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.861 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.861 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.861 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.881 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.881 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.881 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.881 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.881 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.881 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.921 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.921 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.921 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.921 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.921 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.921 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.981 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:48.981 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.981 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.981 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:48.981 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.981 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.981 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:48.982 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:48.999 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:48.999 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.000 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.000 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.000 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.000 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.000 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.000 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.021 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.021 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.021 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.021 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.022 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.022 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.022 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.022 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.022 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [223ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [223ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@68436019(state=PENDING, result=null, callback=(continuation of Deferred@1960517851 after retry RPC after error@1581047494) -> (continuation of Deferred@288957294 after retry RPC after error@931855522) -> (continuation of Deferred@444260208 after retry RPC after error@579401751) -> (continuation of Deferred@2062716283 after retry RPC after error@452788587) -> (continuation of Deferred@1454431259 after retry RPC after error@865980058) -> (continuation of Deferred@1575621831 after retry RPC after error@804171848) -> (continuation of Deferred@2146365866 after retry RPC after error@576330195) -> (continuation of Deferred@790326643 after retry RPC after error@1974330408), errback=(continuation of Deferred@1960517851 after retry RPC after error@1581047494) -> (continuation of Deferred@288957294 after retry RPC after error@931855522) -> (continuation of Deferred@444260208 after retry RPC after error@579401751) -> (continuation of Deferred@2062716283 after retry RPC after error@452788587) -> (continuation of Deferred@1454431259 after retry RPC after error@865980058) -> (continuation of Deferred@1575621831 after retry RPC after error@804171848) -> (continuation of Deferred@2146365866 after retry RPC after error@576330195) -> (continuation of Deferred@790326643 after retry RPC after error@1974330408)))
02:59:49.022 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [223ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [223ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@68436019(state=PENDING, result=null, callback=(continuation of Deferred@1960517851 after retry RPC after error@1581047494) -> (continuation of Deferred@288957294 after retry RPC after error@931855522) -> (continuation of Deferred@444260208 after retry RPC after error@579401751) -> (continuation of Deferred@2062716283 after retry RPC after error@452788587) -> (continuation of Deferred@1454431259 after retry RPC after error@865980058) -> (continuation of Deferred@1575621831 after retry RPC after error@804171848) -> (continuation of Deferred@2146365866 after retry RPC after error@576330195) -> (continuation of Deferred@790326643 after retry RPC after error@1974330408), errback=(continuation of Deferred@1960517851 after retry RPC after error@1581047494) -> (continuation of Deferred@288957294 after retry RPC after error@931855522) -> (continuation of Deferred@444260208 after retry RPC after error@579401751) -> (continuation of Deferred@2062716283 after retry RPC after error@452788587) -> (continuation of Deferred@1454431259 after retry RPC after error@865980058) -> (continuation of Deferred@1575621831 after retry RPC after error@804171848) -> (continuation of Deferred@2146365866 after retry RPC after error@576330195) -> (continuation of Deferred@790326643 after retry RPC after error@1974330408)))
02:59:49.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.041 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.041 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.041 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.041 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.041 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.041 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.041 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.042 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.059 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.060 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.060 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.060 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.060 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.060 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.061 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.080 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.080 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.081 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.081 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.081 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.081 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.081 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.100 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.100 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.100 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.100 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.101 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.101 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.120 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.120 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.121 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.121 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.121 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.121 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.140 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.140 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.140 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.141 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.141 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.141 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.141 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.201 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.201 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.202 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.202 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.202 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.202 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.202 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.202 [DEBUG - kudu-nio-6] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1231457422(state=PENDING, result=null, callback=(continuation of Deferred@323760348 after retry RPC after error@714573340) -> (continuation of Deferred@1224118569 after retry RPC after error@898201157) -> (continuation of Deferred@1194425080 after retry RPC after error@1358996324) -> (continuation of Deferred@755175348 after retry RPC after error@1405513487) -> (continuation of Deferred@527182133 after retry RPC after error@1260626495) -> (continuation of Deferred@2120772160 after retry RPC after error@506911893), errback=(continuation of Deferred@323760348 after retry RPC after error@714573340) -> (continuation of Deferred@1224118569 after retry RPC after error@898201157) -> (continuation of Deferred@1194425080 after retry RPC after error@1358996324) -> (continuation of Deferred@755175348 after retry RPC after error@1405513487) -> (continuation of Deferred@527182133 after retry RPC after error@1260626495) -> (continuation of Deferred@2120772160 after retry RPC after error@506911893)))
02:59:49.203 [DEBUG - kudu-nio-6] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1231457422(state=PENDING, result=null, callback=(continuation of Deferred@323760348 after retry RPC after error@714573340) -> (continuation of Deferred@1224118569 after retry RPC after error@898201157) -> (continuation of Deferred@1194425080 after retry RPC after error@1358996324) -> (continuation of Deferred@755175348 after retry RPC after error@1405513487) -> (continuation of Deferred@527182133 after retry RPC after error@1260626495) -> (continuation of Deferred@2120772160 after retry RPC after error@506911893), errback=(continuation of Deferred@323760348 after retry RPC after error@714573340) -> (continuation of Deferred@1224118569 after retry RPC after error@898201157) -> (continuation of Deferred@1194425080 after retry RPC after error@1358996324) -> (continuation of Deferred@755175348 after retry RPC after error@1405513487) -> (continuation of Deferred@527182133 after retry RPC after error@1260626495) -> (continuation of Deferred@2120772160 after retry RPC after error@506911893)))
02:59:49.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.221 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.221 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.221 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.221 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.221 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.221 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.221 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.222 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.239 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.240 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.240 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.240 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.240 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.240 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.240 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.241 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.259 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.260 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.260 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.260 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.260 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.260 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.260 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.261 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.279 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.280 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.280 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.280 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.280 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.281 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.281 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.300 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.300 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.301 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.301 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.301 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.301 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.320 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.320 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.320 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.321 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.321 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.321 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:49.324195 21227 raft_consensus.cc:534] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 56099a6022da440a81f20e6fe8903fb0)
02:59:49.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:49.324471 21227 raft_consensus.cc:556] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:49.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:49.325351 21227 leader_election.cc:290] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
02:59:49.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:49.328063 20381 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:49.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:49.328281 20378 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111): Network error: Client connection negotiation failed: client connection to 127.19.203.254:46111: connect: Connection refused (error 111)
02:59:49.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:49.328431 20378 leader_election.cc:304] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c; no voters: 56099a6022da440a81f20e6fe8903fb0, efb4c69b20bd4f9692d496c33ddbb24e
02:59:49.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:49.328827 21227 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:59:49.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.340 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.341 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.341 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.341 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.341 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.341 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.401 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.401 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.401 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.401 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.401 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.401 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.402 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2097919359(state=PENDING, result=null, callback=(continuation of Deferred@138622601 after retry RPC after error@1162641976) -> (continuation of Deferred@1773914821 after retry RPC after error@1228614336) -> (continuation of Deferred@1490922000 after retry RPC after error@2032254577) -> (continuation of Deferred@1316728434 after retry RPC after error@1585897559) -> (continuation of Deferred@1807637693 after retry RPC after error@530101645) -> (continuation of Deferred@201560114 after retry RPC after error@1701727677) -> (continuation of Deferred@2091695641 after retry RPC after error@650270940), errback=(continuation of Deferred@138622601 after retry RPC after error@1162641976) -> (continuation of Deferred@1773914821 after retry RPC after error@1228614336) -> (continuation of Deferred@1490922000 after retry RPC after error@2032254577) -> (continuation of Deferred@1316728434 after retry RPC after error@1585897559) -> (continuation of Deferred@1807637693 after retry RPC after error@530101645) -> (continuation of Deferred@201560114 after retry RPC after error@1701727677) -> (continuation of Deferred@2091695641 after retry RPC after error@650270940)))
02:59:49.402 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2097919359(state=PENDING, result=null, callback=(continuation of Deferred@138622601 after retry RPC after error@1162641976) -> (continuation of Deferred@1773914821 after retry RPC after error@1228614336) -> (continuation of Deferred@1490922000 after retry RPC after error@2032254577) -> (continuation of Deferred@1316728434 after retry RPC after error@1585897559) -> (continuation of Deferred@1807637693 after retry RPC after error@530101645) -> (continuation of Deferred@201560114 after retry RPC after error@1701727677) -> (continuation of Deferred@2091695641 after retry RPC after error@650270940), errback=(continuation of Deferred@138622601 after retry RPC after error@1162641976) -> (continuation of Deferred@1773914821 after retry RPC after error@1228614336) -> (continuation of Deferred@1490922000 after retry RPC after error@2032254577) -> (continuation of Deferred@1316728434 after retry RPC after error@1585897559) -> (continuation of Deferred@1807637693 after retry RPC after error@530101645) -> (continuation of Deferred@201560114 after retry RPC after error@1701727677) -> (continuation of Deferred@2091695641 after retry RPC after error@650270940)))
02:59:49.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.421 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.421 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.421 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.421 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.421 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.421 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.422 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.439 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.440 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.440 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.441 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.441 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.441 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.441 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.459 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.461 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.461 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.461 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.461 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.461 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.461 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.461 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.479 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.480 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.480 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.480 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.481 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.481 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.481 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.481 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.499 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.500 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.500 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.500 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.500 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.500 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.500 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.519 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.520 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.520 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.520 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.520 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.520 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.520 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.522 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.600 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.601 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.601 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.601 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.601 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.601 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.602 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.602 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1017132297(state=PENDING, result=null, callback=(continuation of Deferred@1041325863 after retry RPC after error@203887758) -> (continuation of Deferred@264972574 after retry RPC after error@855530899) -> (continuation of Deferred@102945932 after retry RPC after error@848638882) -> (continuation of Deferred@1681944375 after retry RPC after error@613594345) -> (continuation of Deferred@872714819 after retry RPC after error@798469954) -> (continuation of Deferred@1221149776 after retry RPC after error@2043511900), errback=(continuation of Deferred@1041325863 after retry RPC after error@203887758) -> (continuation of Deferred@264972574 after retry RPC after error@855530899) -> (continuation of Deferred@102945932 after retry RPC after error@848638882) -> (continuation of Deferred@1681944375 after retry RPC after error@613594345) -> (continuation of Deferred@872714819 after retry RPC after error@798469954) -> (continuation of Deferred@1221149776 after retry RPC after error@2043511900)))
02:59:49.602 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1017132297(state=PENDING, result=null, callback=(continuation of Deferred@1041325863 after retry RPC after error@203887758) -> (continuation of Deferred@264972574 after retry RPC after error@855530899) -> (continuation of Deferred@102945932 after retry RPC after error@848638882) -> (continuation of Deferred@1681944375 after retry RPC after error@613594345) -> (continuation of Deferred@872714819 after retry RPC after error@798469954) -> (continuation of Deferred@1221149776 after retry RPC after error@2043511900), errback=(continuation of Deferred@1041325863 after retry RPC after error@203887758) -> (continuation of Deferred@264972574 after retry RPC after error@855530899) -> (continuation of Deferred@102945932 after retry RPC after error@848638882) -> (continuation of Deferred@1681944375 after retry RPC after error@613594345) -> (continuation of Deferred@872714819 after retry RPC after error@798469954) -> (continuation of Deferred@1221149776 after retry RPC after error@2043511900)))
02:59:49.619 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.620 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.620 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.620 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.620 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.621 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.621 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.621 [WARN - pool-40-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.640 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.640 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.641 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.641 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.641 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.641 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.641 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.641 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.642 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.661 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.661 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.661 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.661 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.661 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.661 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.681 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.681 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.681 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.681 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.681 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.681 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.682 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.700 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.700 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.701 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.701 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.701 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.701 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.720 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.720 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.720 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.721 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.721 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.721 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.721 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.740 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.740 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.740 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.740 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.740 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.740 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.861 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.861 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.861 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.861 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.861 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.861 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.861 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.862 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.862 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1330894671(state=PENDING, result=null, callback=(continuation of Deferred@1419774761 after retry RPC after error@1067234825) -> (continuation of Deferred@1870017852 after retry RPC after error@1045327410) -> (continuation of Deferred@1784927121 after retry RPC after error@806689834) -> (continuation of Deferred@1855193281 after retry RPC after error@602332383) -> (continuation of Deferred@1475177577 after retry RPC after error@1325522674) -> (continuation of Deferred@31280603 after retry RPC after error@340289819) -> (continuation of Deferred@687526367 after retry RPC after error@1766685820), errback=(continuation of Deferred@1419774761 after retry RPC after error@1067234825) -> (continuation of Deferred@1870017852 after retry RPC after error@1045327410) -> (continuation of Deferred@1784927121 after retry RPC after error@806689834) -> (continuation of Deferred@1855193281 after retry RPC after error@602332383) -> (continuation of Deferred@1475177577 after retry RPC after error@1325522674) -> (continuation of Deferred@31280603 after retry RPC after error@340289819) -> (continuation of Deferred@687526367 after retry RPC after error@1766685820)))
02:59:49.862 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1330894671(state=PENDING, result=null, callback=(continuation of Deferred@1419774761 after retry RPC after error@1067234825) -> (continuation of Deferred@1870017852 after retry RPC after error@1045327410) -> (continuation of Deferred@1784927121 after retry RPC after error@806689834) -> (continuation of Deferred@1855193281 after retry RPC after error@602332383) -> (continuation of Deferred@1475177577 after retry RPC after error@1325522674) -> (continuation of Deferred@31280603 after retry RPC after error@340289819) -> (continuation of Deferred@687526367 after retry RPC after error@1766685820), errback=(continuation of Deferred@1419774761 after retry RPC after error@1067234825) -> (continuation of Deferred@1870017852 after retry RPC after error@1045327410) -> (continuation of Deferred@1784927121 after retry RPC after error@806689834) -> (continuation of Deferred@1855193281 after retry RPC after error@602332383) -> (continuation of Deferred@1475177577 after retry RPC after error@1325522674) -> (continuation of Deferred@31280603 after retry RPC after error@340289819) -> (continuation of Deferred@687526367 after retry RPC after error@1766685820)))
02:59:49.879 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.880 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.880 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.880 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.880 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.880 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.881 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.899 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.900 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.900 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.900 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.900 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.900 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.901 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.919 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.920 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.920 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.921 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.921 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.921 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.921 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.939 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.940 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.940 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.940 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.940 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.940 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.941 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.959 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:49.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:49.960 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.960 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.960 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.960 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:49.960 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.960 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:49.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:49.999 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.000 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.000 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.000 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.001 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.001 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.001 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.019 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.020 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.020 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.021 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.021 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.021 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.021 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.121 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.121 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.121 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.121 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.121 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.121 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.121 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.122 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.122 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@23142812(state=PENDING, result=null, callback=(continuation of Deferred@1138108973 after retry RPC after error@763182771) -> (continuation of Deferred@539347086 after retry RPC after error@683700439) -> (continuation of Deferred@1521856672 after retry RPC after error@511280611) -> (continuation of Deferred@278286106 after retry RPC after error@1006355223) -> (continuation of Deferred@1287406712 after retry RPC after error@515409553) -> (continuation of Deferred@1022686403 after retry RPC after error@1509063473) -> (continuation of Deferred@2009295859 after retry RPC after error@764802964), errback=(continuation of Deferred@1138108973 after retry RPC after error@763182771) -> (continuation of Deferred@539347086 after retry RPC after error@683700439) -> (continuation of Deferred@1521856672 after retry RPC after error@511280611) -> (continuation of Deferred@278286106 after retry RPC after error@1006355223) -> (continuation of Deferred@1287406712 after retry RPC after error@515409553) -> (continuation of Deferred@1022686403 after retry RPC after error@1509063473) -> (continuation of Deferred@2009295859 after retry RPC after error@764802964)))
02:59:50.122 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@23142812(state=PENDING, result=null, callback=(continuation of Deferred@1138108973 after retry RPC after error@763182771) -> (continuation of Deferred@539347086 after retry RPC after error@683700439) -> (continuation of Deferred@1521856672 after retry RPC after error@511280611) -> (continuation of Deferred@278286106 after retry RPC after error@1006355223) -> (continuation of Deferred@1287406712 after retry RPC after error@515409553) -> (continuation of Deferred@1022686403 after retry RPC after error@1509063473) -> (continuation of Deferred@2009295859 after retry RPC after error@764802964), errback=(continuation of Deferred@1138108973 after retry RPC after error@763182771) -> (continuation of Deferred@539347086 after retry RPC after error@683700439) -> (continuation of Deferred@1521856672 after retry RPC after error@511280611) -> (continuation of Deferred@278286106 after retry RPC after error@1006355223) -> (continuation of Deferred@1287406712 after retry RPC after error@515409553) -> (continuation of Deferred@1022686403 after retry RPC after error@1509063473) -> (continuation of Deferred@2009295859 after retry RPC after error@764802964)))
02:59:50.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.141 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.141 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.141 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.141 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.141 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.141 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.141 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.142 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.161 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.161 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.161 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.161 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.161 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.161 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.162 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.162 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.179 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.180 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.180 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.180 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.180 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.180 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.181 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.199 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.200 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.200 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.200 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.200 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.200 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.200 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.219 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.220 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.220 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.220 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.220 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.220 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.220 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.260 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.260 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.261 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.261 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.261 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.261 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.261 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.321 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.321 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.321 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.321 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.321 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.321 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.322 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.341 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.341 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.341 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.341 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.341 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.341 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.342 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1183724556(state=PENDING, result=null, callback=(continuation of Deferred@1446216671 after retry RPC after error@689277652) -> (continuation of Deferred@262663645 after retry RPC after error@952010353) -> (continuation of Deferred@1918486466 after retry RPC after error@2006950007) -> (continuation of Deferred@1206284892 after retry RPC after error@187169247) -> (continuation of Deferred@939280558 after retry RPC after error@1030412862) -> (continuation of Deferred@1231694685 after retry RPC after error@2109201070) -> (continuation of Deferred@677069966 after retry RPC after error@1124843665), errback=(continuation of Deferred@1446216671 after retry RPC after error@689277652) -> (continuation of Deferred@262663645 after retry RPC after error@952010353) -> (continuation of Deferred@1918486466 after retry RPC after error@2006950007) -> (continuation of Deferred@1206284892 after retry RPC after error@187169247) -> (continuation of Deferred@939280558 after retry RPC after error@1030412862) -> (continuation of Deferred@1231694685 after retry RPC after error@2109201070) -> (continuation of Deferred@677069966 after retry RPC after error@1124843665)))
02:59:50.342 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1183724556(state=PENDING, result=null, callback=(continuation of Deferred@1446216671 after retry RPC after error@689277652) -> (continuation of Deferred@262663645 after retry RPC after error@952010353) -> (continuation of Deferred@1918486466 after retry RPC after error@2006950007) -> (continuation of Deferred@1206284892 after retry RPC after error@187169247) -> (continuation of Deferred@939280558 after retry RPC after error@1030412862) -> (continuation of Deferred@1231694685 after retry RPC after error@2109201070) -> (continuation of Deferred@677069966 after retry RPC after error@1124843665), errback=(continuation of Deferred@1446216671 after retry RPC after error@689277652) -> (continuation of Deferred@262663645 after retry RPC after error@952010353) -> (continuation of Deferred@1918486466 after retry RPC after error@2006950007) -> (continuation of Deferred@1206284892 after retry RPC after error@187169247) -> (continuation of Deferred@939280558 after retry RPC after error@1030412862) -> (continuation of Deferred@1231694685 after retry RPC after error@2109201070) -> (continuation of Deferred@677069966 after retry RPC after error@1124843665)))
02:59:50.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.361 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.361 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.361 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.361 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.361 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.361 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.381 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.381 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.381 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.381 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.381 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.381 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.401 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.401 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.401 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.401 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.401 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.401 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.402 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.420 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.421 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.421 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.421 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.421 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.421 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.421 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.440 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.440 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.440 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.441 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.441 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.441 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.481 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.481 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.481 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.481 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.481 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.481 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.481 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.481 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.561 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.561 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.561 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.561 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.561 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.561 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.561 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.562 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@891799530(state=PENDING, result=null, callback=(continuation of Deferred@980032328 after retry RPC after error@771167905) -> (continuation of Deferred@891540777 after retry RPC after error@1043356912) -> (continuation of Deferred@1416278053 after retry RPC after error@1102968343) -> (continuation of Deferred@1780279700 after retry RPC after error@1450700795) -> (continuation of Deferred@1461552776 after retry RPC after error@2042548018) -> (continuation of Deferred@1952696792 after retry RPC after error@486733976), errback=(continuation of Deferred@980032328 after retry RPC after error@771167905) -> (continuation of Deferred@891540777 after retry RPC after error@1043356912) -> (continuation of Deferred@1416278053 after retry RPC after error@1102968343) -> (continuation of Deferred@1780279700 after retry RPC after error@1450700795) -> (continuation of Deferred@1461552776 after retry RPC after error@2042548018) -> (continuation of Deferred@1952696792 after retry RPC after error@486733976)))
02:59:50.562 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@891799530(state=PENDING, result=null, callback=(continuation of Deferred@980032328 after retry RPC after error@771167905) -> (continuation of Deferred@891540777 after retry RPC after error@1043356912) -> (continuation of Deferred@1416278053 after retry RPC after error@1102968343) -> (continuation of Deferred@1780279700 after retry RPC after error@1450700795) -> (continuation of Deferred@1461552776 after retry RPC after error@2042548018) -> (continuation of Deferred@1952696792 after retry RPC after error@486733976), errback=(continuation of Deferred@980032328 after retry RPC after error@771167905) -> (continuation of Deferred@891540777 after retry RPC after error@1043356912) -> (continuation of Deferred@1416278053 after retry RPC after error@1102968343) -> (continuation of Deferred@1780279700 after retry RPC after error@1450700795) -> (continuation of Deferred@1461552776 after retry RPC after error@2042548018) -> (continuation of Deferred@1952696792 after retry RPC after error@486733976)))
02:59:50.579 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.580 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.580 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.580 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.580 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.580 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.580 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.599 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.600 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.600 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.600 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.600 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.600 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.600 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.619 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.620 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.620 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.620 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.620 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.620 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.620 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.639 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.640 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.640 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.640 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.640 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.640 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.641 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.659 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.660 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.660 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.660 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.660 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.661 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.661 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.679 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.680 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.680 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.680 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.680 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.680 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.680 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.699 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.700 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.700 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.700 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.700 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.700 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.700 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.760 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.761 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.761 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.761 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.761 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.761 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.762 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1012786621(state=PENDING, result=null, callback=(continuation of Deferred@1872924937 after retry RPC after error@917849060) -> (continuation of Deferred@406437265 after retry RPC after error@1260045094) -> (continuation of Deferred@818624917 after retry RPC after error@754837003) -> (continuation of Deferred@2046624560 after retry RPC after error@1684270298) -> (continuation of Deferred@227374674 after retry RPC after error@1317213270) -> (continuation of Deferred@1601600053 after retry RPC after error@604522557) -> (continuation of Deferred@1121031432 after retry RPC after error@346425504), errback=(continuation of Deferred@1872924937 after retry RPC after error@917849060) -> (continuation of Deferred@406437265 after retry RPC after error@1260045094) -> (continuation of Deferred@818624917 after retry RPC after error@754837003) -> (continuation of Deferred@2046624560 after retry RPC after error@1684270298) -> (continuation of Deferred@227374674 after retry RPC after error@1317213270) -> (continuation of Deferred@1601600053 after retry RPC after error@604522557) -> (continuation of Deferred@1121031432 after retry RPC after error@346425504)))
02:59:50.762 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1012786621(state=PENDING, result=null, callback=(continuation of Deferred@1872924937 after retry RPC after error@917849060) -> (continuation of Deferred@406437265 after retry RPC after error@1260045094) -> (continuation of Deferred@818624917 after retry RPC after error@754837003) -> (continuation of Deferred@2046624560 after retry RPC after error@1684270298) -> (continuation of Deferred@227374674 after retry RPC after error@1317213270) -> (continuation of Deferred@1601600053 after retry RPC after error@604522557) -> (continuation of Deferred@1121031432 after retry RPC after error@346425504), errback=(continuation of Deferred@1872924937 after retry RPC after error@917849060) -> (continuation of Deferred@406437265 after retry RPC after error@1260045094) -> (continuation of Deferred@818624917 after retry RPC after error@754837003) -> (continuation of Deferred@2046624560 after retry RPC after error@1684270298) -> (continuation of Deferred@227374674 after retry RPC after error@1317213270) -> (continuation of Deferred@1601600053 after retry RPC after error@604522557) -> (continuation of Deferred@1121031432 after retry RPC after error@346425504)))
02:59:50.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.780 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.780 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.781 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.781 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.781 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.781 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.782 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.800 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.800 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.800 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.801 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.801 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.801 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.820 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.820 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.821 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.821 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.821 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.821 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.841 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.841 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.841 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.841 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.841 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.841 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.860 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.860 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.860 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.860 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.860 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.861 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.879 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.880 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.880 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.880 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.880 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.880 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.881 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:50.899 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:50.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:50.900 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.900 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.900 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:50.901 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.901 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.901 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:50.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.021 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.021 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.021 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.021 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.021 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.021 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.021 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.021 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.021 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.022 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.023 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.023 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [3ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] refreshing cache from master, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [244ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1731378514(state=PENDING, result=null, callback=(continuation of Deferred@1840265462 after retry RPC after error@344880256) -> (continuation of Deferred@1986083894 after retry RPC after error@80463749) -> (continuation of Deferred@1758476386 after retry RPC after error@261974924) -> (continuation of Deferred@1885497029 after retry RPC after error@718237859) -> (continuation of Deferred@1127567024 after retry RPC after error@655400017) -> (continuation of Deferred@1143439928 after retry RPC after error@356296823) -> (continuation of Deferred@2081760430 after retry RPC after error@1598344553), errback=(continuation of Deferred@1840265462 after retry RPC after error@344880256) -> (continuation of Deferred@1986083894 after retry RPC after error@80463749) -> (continuation of Deferred@1758476386 after retry RPC after error@261974924) -> (continuation of Deferred@1885497029 after retry RPC after error@718237859) -> (continuation of Deferred@1127567024 after retry RPC after error@655400017) -> (continuation of Deferred@1143439928 after retry RPC after error@356296823) -> (continuation of Deferred@2081760430 after retry RPC after error@1598344553)))
02:59:51.023 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [3ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] refreshing cache from master, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [244ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1731378514(state=PENDING, result=null, callback=(continuation of Deferred@1840265462 after retry RPC after error@344880256) -> (continuation of Deferred@1986083894 after retry RPC after error@80463749) -> (continuation of Deferred@1758476386 after retry RPC after error@261974924) -> (continuation of Deferred@1885497029 after retry RPC after error@718237859) -> (continuation of Deferred@1127567024 after retry RPC after error@655400017) -> (continuation of Deferred@1143439928 after retry RPC after error@356296823) -> (continuation of Deferred@2081760430 after retry RPC after error@1598344553), errback=(continuation of Deferred@1840265462 after retry RPC after error@344880256) -> (continuation of Deferred@1986083894 after retry RPC after error@80463749) -> (continuation of Deferred@1758476386 after retry RPC after error@261974924) -> (continuation of Deferred@1885497029 after retry RPC after error@718237859) -> (continuation of Deferred@1127567024 after retry RPC after error@655400017) -> (continuation of Deferred@1143439928 after retry RPC after error@356296823) -> (continuation of Deferred@2081760430 after retry RPC after error@1598344553)))
02:59:51.039 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.040 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.040 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.040 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.040 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.040 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.040 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.041 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.061 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.061 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.061 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.061 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.061 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.061 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.061 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.062 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.081 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.081 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.081 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.081 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.081 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.081 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.081 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.082 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.101 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.101 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.101 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.101 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.101 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.101 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.101 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.102 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.119 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.120 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.120 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.120 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.120 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.120 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.120 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.160 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.161 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.161 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.161 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.161 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.161 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.221 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.221 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.221 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.221 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.221 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.221 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.222 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.222 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@156853536(state=PENDING, result=null, callback=(continuation of Deferred@1886260039 after retry RPC after error@1999814962) -> (continuation of Deferred@1008837302 after retry RPC after error@2115215479) -> (continuation of Deferred@2124921025 after retry RPC after error@528342390) -> (continuation of Deferred@788064317 after retry RPC after error@1301013901) -> (continuation of Deferred@245972301 after retry RPC after error@2135869315) -> (continuation of Deferred@371093847 after retry RPC after error@78334356), errback=(continuation of Deferred@1886260039 after retry RPC after error@1999814962) -> (continuation of Deferred@1008837302 after retry RPC after error@2115215479) -> (continuation of Deferred@2124921025 after retry RPC after error@528342390) -> (continuation of Deferred@788064317 after retry RPC after error@1301013901) -> (continuation of Deferred@245972301 after retry RPC after error@2135869315) -> (continuation of Deferred@371093847 after retry RPC after error@78334356)))
02:59:51.222 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@156853536(state=PENDING, result=null, callback=(continuation of Deferred@1886260039 after retry RPC after error@1999814962) -> (continuation of Deferred@1008837302 after retry RPC after error@2115215479) -> (continuation of Deferred@2124921025 after retry RPC after error@528342390) -> (continuation of Deferred@788064317 after retry RPC after error@1301013901) -> (continuation of Deferred@245972301 after retry RPC after error@2135869315) -> (continuation of Deferred@371093847 after retry RPC after error@78334356), errback=(continuation of Deferred@1886260039 after retry RPC after error@1999814962) -> (continuation of Deferred@1008837302 after retry RPC after error@2115215479) -> (continuation of Deferred@2124921025 after retry RPC after error@528342390) -> (continuation of Deferred@788064317 after retry RPC after error@1301013901) -> (continuation of Deferred@245972301 after retry RPC after error@2135869315) -> (continuation of Deferred@371093847 after retry RPC after error@78334356)))
02:59:51.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.241 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.241 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.241 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.241 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.241 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.241 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.242 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.261 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.261 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.261 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.261 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.261 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.261 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.261 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.262 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.279 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.280 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.280 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.280 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.280 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.280 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.280 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.301 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.301 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.301 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.301 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.301 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.301 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.301 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.302 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.321 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.321 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.321 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.321 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.321 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.321 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.321 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.322 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.360 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.360 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.361 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.361 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.361 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.361 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.401 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.401 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.401 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.401 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.401 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.401 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.461 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.461 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.461 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.461 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.461 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.461 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.461 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.462 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.462 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1215495120(state=PENDING, result=null, callback=(continuation of Deferred@622501268 after retry RPC after error@659178973) -> (continuation of Deferred@870955924 after retry RPC after error@1813674562) -> (continuation of Deferred@496872552 after retry RPC after error@1754035883) -> (continuation of Deferred@373769525 after retry RPC after error@699459029) -> (continuation of Deferred@1497297511 after retry RPC after error@1384967877) -> (continuation of Deferred@2127460172 after retry RPC after error@1358871309) -> (continuation of Deferred@2130564799 after retry RPC after error@1270019427), errback=(continuation of Deferred@622501268 after retry RPC after error@659178973) -> (continuation of Deferred@870955924 after retry RPC after error@1813674562) -> (continuation of Deferred@496872552 after retry RPC after error@1754035883) -> (continuation of Deferred@373769525 after retry RPC after error@699459029) -> (continuation of Deferred@1497297511 after retry RPC after error@1384967877) -> (continuation of Deferred@2127460172 after retry RPC after error@1358871309) -> (continuation of Deferred@2130564799 after retry RPC after error@1270019427)))
02:59:51.462 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1215495120(state=PENDING, result=null, callback=(continuation of Deferred@622501268 after retry RPC after error@659178973) -> (continuation of Deferred@870955924 after retry RPC after error@1813674562) -> (continuation of Deferred@496872552 after retry RPC after error@1754035883) -> (continuation of Deferred@373769525 after retry RPC after error@699459029) -> (continuation of Deferred@1497297511 after retry RPC after error@1384967877) -> (continuation of Deferred@2127460172 after retry RPC after error@1358871309) -> (continuation of Deferred@2130564799 after retry RPC after error@1270019427), errback=(continuation of Deferred@622501268 after retry RPC after error@659178973) -> (continuation of Deferred@870955924 after retry RPC after error@1813674562) -> (continuation of Deferred@496872552 after retry RPC after error@1754035883) -> (continuation of Deferred@373769525 after retry RPC after error@699459029) -> (continuation of Deferred@1497297511 after retry RPC after error@1384967877) -> (continuation of Deferred@2127460172 after retry RPC after error@1358871309) -> (continuation of Deferred@2130564799 after retry RPC after error@1270019427)))
02:59:51.479 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.480 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.480 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.480 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.480 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.480 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.480 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.481 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.501 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.501 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.501 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.501 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.501 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.501 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.501 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.502 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.521 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.521 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.521 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.521 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.521 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.521 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.521 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.522 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.539 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.540 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.540 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.540 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.541 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.541 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.541 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.559 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.560 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.560 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.560 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.560 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.560 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.560 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.561 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.579 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.580 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.580 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.580 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.580 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.580 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.580 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.599 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.600 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.600 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.600 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.600 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.600 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.600 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.660 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.660 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.660 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.660 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.661 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.661 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.662 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@897493254(state=PENDING, result=null, callback=(continuation of Deferred@352603978 after retry RPC after error@1721927883) -> (continuation of Deferred@1726306776 after retry RPC after error@638839451) -> (continuation of Deferred@291147532 after retry RPC after error@1665972540) -> (continuation of Deferred@799741716 after retry RPC after error@374695752) -> (continuation of Deferred@1404221537 after retry RPC after error@1437691506) -> (continuation of Deferred@584437863 after retry RPC after error@528976978) -> (continuation of Deferred@1496342131 after retry RPC after error@128739801), errback=(continuation of Deferred@352603978 after retry RPC after error@1721927883) -> (continuation of Deferred@1726306776 after retry RPC after error@638839451) -> (continuation of Deferred@291147532 after retry RPC after error@1665972540) -> (continuation of Deferred@799741716 after retry RPC after error@374695752) -> (continuation of Deferred@1404221537 after retry RPC after error@1437691506) -> (continuation of Deferred@584437863 after retry RPC after error@528976978) -> (continuation of Deferred@1496342131 after retry RPC after error@128739801)))
02:59:51.662 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@897493254(state=PENDING, result=null, callback=(continuation of Deferred@352603978 after retry RPC after error@1721927883) -> (continuation of Deferred@1726306776 after retry RPC after error@638839451) -> (continuation of Deferred@291147532 after retry RPC after error@1665972540) -> (continuation of Deferred@799741716 after retry RPC after error@374695752) -> (continuation of Deferred@1404221537 after retry RPC after error@1437691506) -> (continuation of Deferred@584437863 after retry RPC after error@528976978) -> (continuation of Deferred@1496342131 after retry RPC after error@128739801), errback=(continuation of Deferred@352603978 after retry RPC after error@1721927883) -> (continuation of Deferred@1726306776 after retry RPC after error@638839451) -> (continuation of Deferred@291147532 after retry RPC after error@1665972540) -> (continuation of Deferred@799741716 after retry RPC after error@374695752) -> (continuation of Deferred@1404221537 after retry RPC after error@1437691506) -> (continuation of Deferred@584437863 after retry RPC after error@528976978) -> (continuation of Deferred@1496342131 after retry RPC after error@128739801)))
02:59:51.679 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.680 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.680 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.680 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.680 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.680 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.681 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.699 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.700 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.700 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.700 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.700 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.701 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.701 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.720 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.720 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.720 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.720 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.721 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.721 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.721 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.739 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.740 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.740 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.740 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.741 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.741 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.741 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.759 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.760 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.760 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.761 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.761 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.761 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.761 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.779 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.780 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.780 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.781 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.781 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.781 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.781 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.821 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.821 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.821 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.821 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.821 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.821 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.822 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.881 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.881 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.881 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.881 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.881 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.881 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.882 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.882 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [203ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@54368971(state=PENDING, result=null, callback=(continuation of Deferred@828269603 after retry RPC after error@1434802095) -> (continuation of Deferred@949314815 after retry RPC after error@1460755172) -> (continuation of Deferred@355222361 after retry RPC after error@1793803900) -> (continuation of Deferred@2034918722 after retry RPC after error@617731829) -> (continuation of Deferred@1393586109 after retry RPC after error@1677200133) -> (continuation of Deferred@1236402765 after retry RPC after error@1669202928) -> (continuation of Deferred@860087717 after retry RPC after error@281043179), errback=(continuation of Deferred@828269603 after retry RPC after error@1434802095) -> (continuation of Deferred@949314815 after retry RPC after error@1460755172) -> (continuation of Deferred@355222361 after retry RPC after error@1793803900) -> (continuation of Deferred@2034918722 after retry RPC after error@617731829) -> (continuation of Deferred@1393586109 after retry RPC after error@1677200133) -> (continuation of Deferred@1236402765 after retry RPC after error@1669202928) -> (continuation of Deferred@860087717 after retry RPC after error@281043179)))
02:59:51.882 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [203ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@54368971(state=PENDING, result=null, callback=(continuation of Deferred@828269603 after retry RPC after error@1434802095) -> (continuation of Deferred@949314815 after retry RPC after error@1460755172) -> (continuation of Deferred@355222361 after retry RPC after error@1793803900) -> (continuation of Deferred@2034918722 after retry RPC after error@617731829) -> (continuation of Deferred@1393586109 after retry RPC after error@1677200133) -> (continuation of Deferred@1236402765 after retry RPC after error@1669202928) -> (continuation of Deferred@860087717 after retry RPC after error@281043179), errback=(continuation of Deferred@828269603 after retry RPC after error@1434802095) -> (continuation of Deferred@949314815 after retry RPC after error@1460755172) -> (continuation of Deferred@355222361 after retry RPC after error@1793803900) -> (continuation of Deferred@2034918722 after retry RPC after error@617731829) -> (continuation of Deferred@1393586109 after retry RPC after error@1677200133) -> (continuation of Deferred@1236402765 after retry RPC after error@1669202928) -> (continuation of Deferred@860087717 after retry RPC after error@281043179)))
02:59:51.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.901 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.901 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.901 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.901 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.901 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.901 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.921 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.921 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.921 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.921 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.921 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.921 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.922 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.941 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.941 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.941 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.941 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.941 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.941 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.942 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.961 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.961 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.961 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.961 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.961 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.961 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.962 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:51.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:51.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:51.981 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.981 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.981 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.981 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.981 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:51.981 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:51.982 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.001 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.001 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.001 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.001 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.001 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.001 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.002 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.059 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.060 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.060 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.060 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.060 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.060 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.060 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.061 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1988754353(state=PENDING, result=null, callback=(continuation of Deferred@1700600499 after retry RPC after error@1638298052) -> (continuation of Deferred@1643243827 after retry RPC after error@1478283983) -> (continuation of Deferred@1436144637 after retry RPC after error@132358363) -> (continuation of Deferred@1765108069 after retry RPC after error@1459700774) -> (continuation of Deferred@1285317376 after retry RPC after error@1449391920) -> (continuation of Deferred@350208728 after retry RPC after error@1226998801), errback=(continuation of Deferred@1700600499 after retry RPC after error@1638298052) -> (continuation of Deferred@1643243827 after retry RPC after error@1478283983) -> (continuation of Deferred@1436144637 after retry RPC after error@132358363) -> (continuation of Deferred@1765108069 after retry RPC after error@1459700774) -> (continuation of Deferred@1285317376 after retry RPC after error@1449391920) -> (continuation of Deferred@350208728 after retry RPC after error@1226998801)))
02:59:52.062 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1988754353(state=PENDING, result=null, callback=(continuation of Deferred@1700600499 after retry RPC after error@1638298052) -> (continuation of Deferred@1643243827 after retry RPC after error@1478283983) -> (continuation of Deferred@1436144637 after retry RPC after error@132358363) -> (continuation of Deferred@1765108069 after retry RPC after error@1459700774) -> (continuation of Deferred@1285317376 after retry RPC after error@1449391920) -> (continuation of Deferred@350208728 after retry RPC after error@1226998801), errback=(continuation of Deferred@1700600499 after retry RPC after error@1638298052) -> (continuation of Deferred@1643243827 after retry RPC after error@1478283983) -> (continuation of Deferred@1436144637 after retry RPC after error@132358363) -> (continuation of Deferred@1765108069 after retry RPC after error@1459700774) -> (continuation of Deferred@1285317376 after retry RPC after error@1449391920) -> (continuation of Deferred@350208728 after retry RPC after error@1226998801)))
02:59:52.079 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.081 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.081 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.081 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.081 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.081 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.081 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.082 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.101 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.101 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.101 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.101 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.101 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.101 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.102 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.121 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.121 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.121 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.121 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.121 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.121 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.122 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.141 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.141 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.141 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.141 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.141 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.141 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.142 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.161 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.161 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.161 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.161 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.161 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.161 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.181 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.181 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.181 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.181 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.181 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.181 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.182 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.219 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.220 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.220 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.220 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.220 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.220 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.220 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.239 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.240 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.240 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.240 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.240 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.240 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.240 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.241 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.241 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [3ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@898662851(state=PENDING, result=null, callback=(continuation of Deferred@1286445306 after retry RPC after error@1102465627) -> (continuation of Deferred@1776465189 after retry RPC after error@1489874755) -> (continuation of Deferred@2022147816 after retry RPC after error@194051889) -> (continuation of Deferred@1999059195 after retry RPC after error@1118123755) -> (continuation of Deferred@105990289 after retry RPC after error@171202077) -> (continuation of Deferred@1150965568 after retry RPC after error@803414870) -> (continuation of Deferred@595260411 after retry RPC after error@190242103), errback=(continuation of Deferred@1286445306 after retry RPC after error@1102465627) -> (continuation of Deferred@1776465189 after retry RPC after error@1489874755) -> (continuation of Deferred@2022147816 after retry RPC after error@194051889) -> (continuation of Deferred@1999059195 after retry RPC after error@1118123755) -> (continuation of Deferred@105990289 after retry RPC after error@171202077) -> (continuation of Deferred@1150965568 after retry RPC after error@803414870) -> (continuation of Deferred@595260411 after retry RPC after error@190242103)))
02:59:52.242 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [3ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@898662851(state=PENDING, result=null, callback=(continuation of Deferred@1286445306 after retry RPC after error@1102465627) -> (continuation of Deferred@1776465189 after retry RPC after error@1489874755) -> (continuation of Deferred@2022147816 after retry RPC after error@194051889) -> (continuation of Deferred@1999059195 after retry RPC after error@1118123755) -> (continuation of Deferred@105990289 after retry RPC after error@171202077) -> (continuation of Deferred@1150965568 after retry RPC after error@803414870) -> (continuation of Deferred@595260411 after retry RPC after error@190242103), errback=(continuation of Deferred@1286445306 after retry RPC after error@1102465627) -> (continuation of Deferred@1776465189 after retry RPC after error@1489874755) -> (continuation of Deferred@2022147816 after retry RPC after error@194051889) -> (continuation of Deferred@1999059195 after retry RPC after error@1118123755) -> (continuation of Deferred@105990289 after retry RPC after error@171202077) -> (continuation of Deferred@1150965568 after retry RPC after error@803414870) -> (continuation of Deferred@595260411 after retry RPC after error@190242103)))
02:59:52.259 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.260 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.260 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.260 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.261 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.261 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.261 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.261 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.279 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.280 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.280 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.280 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.280 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.280 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.280 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.299 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.300 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.300 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.300 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.300 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.300 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.300 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.321 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.321 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.321 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.321 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.321 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.321 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.322 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.341 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.341 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.341 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.341 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.341 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.341 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.361 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.361 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.361 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.361 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.361 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.361 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.362 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.381 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.381 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.381 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.381 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.381 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.381 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.381 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.382 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.481 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.481 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.481 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.481 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.481 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.481 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.481 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.482 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2057209285(state=PENDING, result=null, callback=(continuation of Deferred@804613871 after retry RPC after error@2662574) -> (continuation of Deferred@72453464 after retry RPC after error@183468218) -> (continuation of Deferred@250152222 after retry RPC after error@1266809277) -> (continuation of Deferred@1135773400 after retry RPC after error@900609092) -> (continuation of Deferred@1932072474 after retry RPC after error@924931539) -> (continuation of Deferred@1618117033 after retry RPC after error@616207863) -> (continuation of Deferred@530112188 after retry RPC after error@581607249), errback=(continuation of Deferred@804613871 after retry RPC after error@2662574) -> (continuation of Deferred@72453464 after retry RPC after error@183468218) -> (continuation of Deferred@250152222 after retry RPC after error@1266809277) -> (continuation of Deferred@1135773400 after retry RPC after error@900609092) -> (continuation of Deferred@1932072474 after retry RPC after error@924931539) -> (continuation of Deferred@1618117033 after retry RPC after error@616207863) -> (continuation of Deferred@530112188 after retry RPC after error@581607249)))
02:59:52.482 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2057209285(state=PENDING, result=null, callback=(continuation of Deferred@804613871 after retry RPC after error@2662574) -> (continuation of Deferred@72453464 after retry RPC after error@183468218) -> (continuation of Deferred@250152222 after retry RPC after error@1266809277) -> (continuation of Deferred@1135773400 after retry RPC after error@900609092) -> (continuation of Deferred@1932072474 after retry RPC after error@924931539) -> (continuation of Deferred@1618117033 after retry RPC after error@616207863) -> (continuation of Deferred@530112188 after retry RPC after error@581607249), errback=(continuation of Deferred@804613871 after retry RPC after error@2662574) -> (continuation of Deferred@72453464 after retry RPC after error@183468218) -> (continuation of Deferred@250152222 after retry RPC after error@1266809277) -> (continuation of Deferred@1135773400 after retry RPC after error@900609092) -> (continuation of Deferred@1932072474 after retry RPC after error@924931539) -> (continuation of Deferred@1618117033 after retry RPC after error@616207863) -> (continuation of Deferred@530112188 after retry RPC after error@581607249)))
02:59:52.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.501 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.501 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.501 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.501 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.501 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.501 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.520 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.520 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.521 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.521 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.521 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.521 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.541 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.541 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.541 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.541 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.541 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.541 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.560 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.561 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.561 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.561 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.561 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.561 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.561 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.579 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.580 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.580 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.580 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.581 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.581 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.581 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.599 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.600 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.600 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.600 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.601 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.601 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.601 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.601 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.659 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.660 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.660 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.660 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.660 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.660 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.660 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.760 [DEBUG - pool-40-thread-1] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@734998530(state=PENDING, result=null, callback=(continuation of Deferred@1058696834 after retry RPC after error@2051826321) -> (continuation of Deferred@204018790 after retry RPC after error@1580493505) -> (continuation of Deferred@842424827 after retry RPC after error@729269702) -> (continuation of Deferred@1866039018 after retry RPC after error@304509870) -> (continuation of Deferred@320795048 after retry RPC after error@106061816) -> (continuation of Deferred@574144502 after retry RPC after error@60953372) -> (continuation of Deferred@1742020977 after retry RPC after error@1944854508), errback=(continuation of Deferred@1058696834 after retry RPC after error@2051826321) -> (continuation of Deferred@204018790 after retry RPC after error@1580493505) -> (continuation of Deferred@842424827 after retry RPC after error@729269702) -> (continuation of Deferred@1866039018 after retry RPC after error@304509870) -> (continuation of Deferred@320795048 after retry RPC after error@106061816) -> (continuation of Deferred@574144502 after retry RPC after error@60953372) -> (continuation of Deferred@1742020977 after retry RPC after error@1944854508)))
02:59:52.761 [WARN - pool-40-thread-1] (HashedWheelTimer.java:729) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
02:59:52.779 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.780 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.780 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.780 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.780 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.780 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.780 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.799 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.800 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.800 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.800 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.801 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.801 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.801 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.819 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.820 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.820 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.820 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.820 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.820 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.820 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.839 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.839 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.840 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.840 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.840 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.840 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.840 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.840 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.861 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.861 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.861 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.861 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.861 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.861 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.862 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.881 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.881 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.881 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.881 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.881 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.881 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.882 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.941 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.941 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.941 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.941 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.941 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.941 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.941 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.942 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.942 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1576970583(state=PENDING, result=null, callback=(continuation of Deferred@1177139694 after retry RPC after error@1011813812) -> (continuation of Deferred@1711102053 after retry RPC after error@1306653084) -> (continuation of Deferred@1183631111 after retry RPC after error@40517839) -> (continuation of Deferred@232464023 after retry RPC after error@1333027745) -> (continuation of Deferred@1876963767 after retry RPC after error@1063024828) -> (continuation of Deferred@1789326950 after retry RPC after error@111405560), errback=(continuation of Deferred@1177139694 after retry RPC after error@1011813812) -> (continuation of Deferred@1711102053 after retry RPC after error@1306653084) -> (continuation of Deferred@1183631111 after retry RPC after error@40517839) -> (continuation of Deferred@232464023 after retry RPC after error@1333027745) -> (continuation of Deferred@1876963767 after retry RPC after error@1063024828) -> (continuation of Deferred@1789326950 after retry RPC after error@111405560)))
02:59:52.942 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1576970583(state=PENDING, result=null, callback=(continuation of Deferred@1177139694 after retry RPC after error@1011813812) -> (continuation of Deferred@1711102053 after retry RPC after error@1306653084) -> (continuation of Deferred@1183631111 after retry RPC after error@40517839) -> (continuation of Deferred@232464023 after retry RPC after error@1333027745) -> (continuation of Deferred@1876963767 after retry RPC after error@1063024828) -> (continuation of Deferred@1789326950 after retry RPC after error@111405560), errback=(continuation of Deferred@1177139694 after retry RPC after error@1011813812) -> (continuation of Deferred@1711102053 after retry RPC after error@1306653084) -> (continuation of Deferred@1183631111 after retry RPC after error@40517839) -> (continuation of Deferred@232464023 after retry RPC after error@1333027745) -> (continuation of Deferred@1876963767 after retry RPC after error@1063024828) -> (continuation of Deferred@1789326950 after retry RPC after error@111405560)))
02:59:52.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.961 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.961 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.961 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.961 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.961 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.961 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.962 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:52.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:52.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:52.981 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.981 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.981 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:52.981 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.981 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.981 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:52.982 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.001 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.001 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.001 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.001 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.001 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.001 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.002 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.021 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.021 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.021 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.021 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.021 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.021 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.041 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.041 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.041 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.041 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.041 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.041 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.042 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.061 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.061 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.061 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.061 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.061 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.061 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.062 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.119 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.120 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.120 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.120 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.120 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.120 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.121 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.180 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.181 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.181 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.181 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.181 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.181 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.181 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@410535806(state=PENDING, result=null, callback=(continuation of Deferred@67647938 after retry RPC after error@50466937) -> (continuation of Deferred@856548143 after retry RPC after error@1762922887) -> (continuation of Deferred@748155726 after retry RPC after error@1057337213) -> (continuation of Deferred@1666000681 after retry RPC after error@706517742) -> (continuation of Deferred@964821344 after retry RPC after error@737034090) -> (continuation of Deferred@563093841 after retry RPC after error@1863648359) -> (continuation of Deferred@1186300388 after retry RPC after error@486208311), errback=(continuation of Deferred@67647938 after retry RPC after error@50466937) -> (continuation of Deferred@856548143 after retry RPC after error@1762922887) -> (continuation of Deferred@748155726 after retry RPC after error@1057337213) -> (continuation of Deferred@1666000681 after retry RPC after error@706517742) -> (continuation of Deferred@964821344 after retry RPC after error@737034090) -> (continuation of Deferred@563093841 after retry RPC after error@1863648359) -> (continuation of Deferred@1186300388 after retry RPC after error@486208311)))
02:59:53.182 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@410535806(state=PENDING, result=null, callback=(continuation of Deferred@67647938 after retry RPC after error@50466937) -> (continuation of Deferred@856548143 after retry RPC after error@1762922887) -> (continuation of Deferred@748155726 after retry RPC after error@1057337213) -> (continuation of Deferred@1666000681 after retry RPC after error@706517742) -> (continuation of Deferred@964821344 after retry RPC after error@737034090) -> (continuation of Deferred@563093841 after retry RPC after error@1863648359) -> (continuation of Deferred@1186300388 after retry RPC after error@486208311), errback=(continuation of Deferred@67647938 after retry RPC after error@50466937) -> (continuation of Deferred@856548143 after retry RPC after error@1762922887) -> (continuation of Deferred@748155726 after retry RPC after error@1057337213) -> (continuation of Deferred@1666000681 after retry RPC after error@706517742) -> (continuation of Deferred@964821344 after retry RPC after error@737034090) -> (continuation of Deferred@563093841 after retry RPC after error@1863648359) -> (continuation of Deferred@1186300388 after retry RPC after error@486208311)))
02:59:53.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.200 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.200 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.200 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.201 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.201 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.201 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.219 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.220 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.220 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.220 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.220 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.220 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.220 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.239 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.240 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.240 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.240 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.240 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.240 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.240 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.241 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.259 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.260 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.260 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.260 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.260 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.260 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.260 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.261 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:53.276437 21254 raft_consensus.cc:534] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 56099a6022da440a81f20e6fe8903fb0)
02:59:53.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:53.276629 21254 raft_consensus.cc:556] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:53.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:53.277501 21254 leader_election.cc:290] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
02:59:53.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:53.278726 20378 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.203.254:46111: connect: Connection refused (error 111) [suppressed 5 similar messages]
02:59:53.279 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:53.280124 20381 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:53.280 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.280 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.280 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.280 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.280 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.280 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:53.280484 20378 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111): Network error: Client connection negotiation failed: client connection to 127.19.203.254:46111: connect: Connection refused (error 111)
02:59:53.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:53.280682 20378 leader_election.cc:304] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c; no voters: 56099a6022da440a81f20e6fe8903fb0, efb4c69b20bd4f9692d496c33ddbb24e
02:59:53.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:53.281152 21254 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:59:53.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.320 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.320 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.320 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.320 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.320 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.320 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.360 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.360 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.360 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.360 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.361 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.361 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.361 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1621713578(state=PENDING, result=null, callback=(continuation of Deferred@362413418 after retry RPC after error@171082019) -> (continuation of Deferred@526986869 after retry RPC after error@880718592) -> (continuation of Deferred@1859545672 after retry RPC after error@1895028008) -> (continuation of Deferred@641685445 after retry RPC after error@1729627825) -> (continuation of Deferred@985773104 after retry RPC after error@1425370325) -> (continuation of Deferred@145262833 after retry RPC after error@799156955), errback=(continuation of Deferred@362413418 after retry RPC after error@171082019) -> (continuation of Deferred@526986869 after retry RPC after error@880718592) -> (continuation of Deferred@1859545672 after retry RPC after error@1895028008) -> (continuation of Deferred@641685445 after retry RPC after error@1729627825) -> (continuation of Deferred@985773104 after retry RPC after error@1425370325) -> (continuation of Deferred@145262833 after retry RPC after error@799156955)))
02:59:53.362 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1621713578(state=PENDING, result=null, callback=(continuation of Deferred@362413418 after retry RPC after error@171082019) -> (continuation of Deferred@526986869 after retry RPC after error@880718592) -> (continuation of Deferred@1859545672 after retry RPC after error@1895028008) -> (continuation of Deferred@641685445 after retry RPC after error@1729627825) -> (continuation of Deferred@985773104 after retry RPC after error@1425370325) -> (continuation of Deferred@145262833 after retry RPC after error@799156955), errback=(continuation of Deferred@362413418 after retry RPC after error@171082019) -> (continuation of Deferred@526986869 after retry RPC after error@880718592) -> (continuation of Deferred@1859545672 after retry RPC after error@1895028008) -> (continuation of Deferred@641685445 after retry RPC after error@1729627825) -> (continuation of Deferred@985773104 after retry RPC after error@1425370325) -> (continuation of Deferred@145262833 after retry RPC after error@799156955)))
02:59:53.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.380 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.380 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.380 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.380 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.381 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.381 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.400 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.400 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.400 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.401 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.401 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.401 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.420 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.420 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.420 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.420 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.420 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.420 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.421 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.439 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.440 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.440 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.440 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.440 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.440 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.440 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.459 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.460 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.460 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.460 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.460 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.461 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.461 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.462 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.501 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.501 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.501 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.501 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.502 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.502 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.502 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.502 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.581 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.581 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.581 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.581 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.581 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.581 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.601 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.601 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.601 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.601 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.601 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.601 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.602 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@644825806(state=PENDING, result=null, callback=(continuation of Deferred@136423848 after retry RPC after error@1963433958) -> (continuation of Deferred@1713345825 after retry RPC after error@985317918) -> (continuation of Deferred@1062420488 after retry RPC after error@1986134187) -> (continuation of Deferred@1825449435 after retry RPC after error@8958062) -> (continuation of Deferred@637465463 after retry RPC after error@1140840507) -> (continuation of Deferred@1577246050 after retry RPC after error@466357516) -> (continuation of Deferred@1241194569 after retry RPC after error@1962415949), errback=(continuation of Deferred@136423848 after retry RPC after error@1963433958) -> (continuation of Deferred@1713345825 after retry RPC after error@985317918) -> (continuation of Deferred@1062420488 after retry RPC after error@1986134187) -> (continuation of Deferred@1825449435 after retry RPC after error@8958062) -> (continuation of Deferred@637465463 after retry RPC after error@1140840507) -> (continuation of Deferred@1577246050 after retry RPC after error@466357516) -> (continuation of Deferred@1241194569 after retry RPC after error@1962415949)))
02:59:53.602 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@644825806(state=PENDING, result=null, callback=(continuation of Deferred@136423848 after retry RPC after error@1963433958) -> (continuation of Deferred@1713345825 after retry RPC after error@985317918) -> (continuation of Deferred@1062420488 after retry RPC after error@1986134187) -> (continuation of Deferred@1825449435 after retry RPC after error@8958062) -> (continuation of Deferred@637465463 after retry RPC after error@1140840507) -> (continuation of Deferred@1577246050 after retry RPC after error@466357516) -> (continuation of Deferred@1241194569 after retry RPC after error@1962415949), errback=(continuation of Deferred@136423848 after retry RPC after error@1963433958) -> (continuation of Deferred@1713345825 after retry RPC after error@985317918) -> (continuation of Deferred@1062420488 after retry RPC after error@1986134187) -> (continuation of Deferred@1825449435 after retry RPC after error@8958062) -> (continuation of Deferred@637465463 after retry RPC after error@1140840507) -> (continuation of Deferred@1577246050 after retry RPC after error@466357516) -> (continuation of Deferred@1241194569 after retry RPC after error@1962415949)))
02:59:53.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.621 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.621 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.621 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.621 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.621 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.621 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.622 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.641 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.641 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.641 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.641 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.641 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.641 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.661 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.661 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.661 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.661 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.661 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.661 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.680 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.680 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.680 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.681 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.681 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.681 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.700 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.700 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.700 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.700 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.700 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.700 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.740 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.740 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.741 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.741 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.741 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.741 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.780 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.781 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.781 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.781 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.781 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.781 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.786 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1466919494(state=PENDING, result=null, callback=(continuation of Deferred@1670995353 after retry RPC after error@383690306) -> (continuation of Deferred@1714714053 after retry RPC after error@1243946527) -> (continuation of Deferred@1668937559 after retry RPC after error@16163276) -> (continuation of Deferred@1168729674 after retry RPC after error@656297222) -> (continuation of Deferred@192009725 after retry RPC after error@1044180382) -> (continuation of Deferred@808015653 after retry RPC after error@1695177307), errback=(continuation of Deferred@1670995353 after retry RPC after error@383690306) -> (continuation of Deferred@1714714053 after retry RPC after error@1243946527) -> (continuation of Deferred@1668937559 after retry RPC after error@16163276) -> (continuation of Deferred@1168729674 after retry RPC after error@656297222) -> (continuation of Deferred@192009725 after retry RPC after error@1044180382) -> (continuation of Deferred@808015653 after retry RPC after error@1695177307)))
02:59:53.786 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1466919494(state=PENDING, result=null, callback=(continuation of Deferred@1670995353 after retry RPC after error@383690306) -> (continuation of Deferred@1714714053 after retry RPC after error@1243946527) -> (continuation of Deferred@1668937559 after retry RPC after error@16163276) -> (continuation of Deferred@1168729674 after retry RPC after error@656297222) -> (continuation of Deferred@192009725 after retry RPC after error@1044180382) -> (continuation of Deferred@808015653 after retry RPC after error@1695177307), errback=(continuation of Deferred@1670995353 after retry RPC after error@383690306) -> (continuation of Deferred@1714714053 after retry RPC after error@1243946527) -> (continuation of Deferred@1668937559 after retry RPC after error@16163276) -> (continuation of Deferred@1168729674 after retry RPC after error@656297222) -> (continuation of Deferred@192009725 after retry RPC after error@1044180382) -> (continuation of Deferred@808015653 after retry RPC after error@1695177307)))
02:59:53.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.800 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.801 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.801 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.801 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.801 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.801 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.820 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.820 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.820 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.820 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.821 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.821 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.822 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.840 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.840 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.840 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.840 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.840 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.841 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.859 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.860 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.860 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.860 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.860 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.860 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.860 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.879 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.880 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.880 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.880 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.880 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.880 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.880 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.919 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.920 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.920 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.920 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.920 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.920 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.920 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:53.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:53.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:53.980 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.980 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.980 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.980 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.981 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:53.981 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:53.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.060 [DEBUG - pool-40-thread-1] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@814932835(state=PENDING, result=null, callback=(continuation of Deferred@538273359 after retry RPC after error@1894173816) -> (continuation of Deferred@1855136406 after retry RPC after error@1228754396) -> (continuation of Deferred@1116827050 after retry RPC after error@72543372) -> (continuation of Deferred@1940060418 after retry RPC after error@1355942151) -> (continuation of Deferred@434619018 after retry RPC after error@172125110) -> (continuation of Deferred@1850534165 after retry RPC after error@773214186) -> (continuation of Deferred@155335472 after retry RPC after error@272957045), errback=(continuation of Deferred@538273359 after retry RPC after error@1894173816) -> (continuation of Deferred@1855136406 after retry RPC after error@1228754396) -> (continuation of Deferred@1116827050 after retry RPC after error@72543372) -> (continuation of Deferred@1940060418 after retry RPC after error@1355942151) -> (continuation of Deferred@434619018 after retry RPC after error@172125110) -> (continuation of Deferred@1850534165 after retry RPC after error@773214186) -> (continuation of Deferred@155335472 after retry RPC after error@272957045)))
02:59:54.060 [WARN - pool-40-thread-1] (HashedWheelTimer.java:729) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
02:59:54.079 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.080 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.080 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.080 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.080 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.080 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.081 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.081 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.100 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.100 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.101 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.101 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.101 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.101 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.121 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.121 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.121 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.121 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.121 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.121 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.141 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.141 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.141 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.141 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.141 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.141 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.141 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.160 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.160 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.160 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.161 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.161 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.161 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.180 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.180 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.180 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.180 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.181 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.181 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.181 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.181 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.182 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.241 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.241 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.241 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.242 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.242 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.242 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.242 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.261 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.261 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.261 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.261 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.261 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.261 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.262 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.339 [DEBUG - pool-40-thread-1] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [163ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@539934129(state=PENDING, result=null, callback=(continuation of Deferred@1625004580 after retry RPC after error@2112397222) -> (continuation of Deferred@1706270773 after retry RPC after error@807754247) -> (continuation of Deferred@202632349 after retry RPC after error@901732118) -> (continuation of Deferred@1307966364 after retry RPC after error@1461347655) -> (continuation of Deferred@1157504061 after retry RPC after error@196535379) -> (continuation of Deferred@1769850786 after retry RPC after error@1841932183) -> (continuation of Deferred@1508004327 after retry RPC after error@1699390196) -> (continuation of Deferred@1653418280 after retry RPC after error@1121505265), errback=(continuation of Deferred@1625004580 after retry RPC after error@2112397222) -> (continuation of Deferred@1706270773 after retry RPC after error@807754247) -> (continuation of Deferred@202632349 after retry RPC after error@901732118) -> (continuation of Deferred@1307966364 after retry RPC after error@1461347655) -> (continuation of Deferred@1157504061 after retry RPC after error@196535379) -> (continuation of Deferred@1769850786 after retry RPC after error@1841932183) -> (continuation of Deferred@1508004327 after retry RPC after error@1699390196) -> (continuation of Deferred@1653418280 after retry RPC after error@1121505265)))
02:59:54.340 [WARN - pool-40-thread-1] (HashedWheelTimer.java:729) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
02:59:54.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.360 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.360 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.360 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.361 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.361 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.361 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.362 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.380 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.380 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.381 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.381 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.381 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.381 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.400 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.400 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.401 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.401 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.401 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.401 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.420 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.420 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.420 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.421 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.421 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.421 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.421 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.440 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.440 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.441 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.441 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.441 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.441 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.442 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.481 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.481 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.481 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.481 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.481 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.481 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.482 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.501 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.501 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.501 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.501 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.501 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.501 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.541 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.541 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.541 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.541 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.541 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.541 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.542 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.542 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [3ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@303172654(state=PENDING, result=null, callback=(continuation of Deferred@794910941 after retry RPC after error@2090247415) -> (continuation of Deferred@970160482 after retry RPC after error@992112906) -> (continuation of Deferred@534874427 after retry RPC after error@1444489004) -> (continuation of Deferred@1956863204 after retry RPC after error@1554106715) -> (continuation of Deferred@1327119269 after retry RPC after error@1353012525) -> (continuation of Deferred@1041201057 after retry RPC after error@1823902250) -> (continuation of Deferred@1928286986 after retry RPC after error@135887943), errback=(continuation of Deferred@794910941 after retry RPC after error@2090247415) -> (continuation of Deferred@970160482 after retry RPC after error@992112906) -> (continuation of Deferred@534874427 after retry RPC after error@1444489004) -> (continuation of Deferred@1956863204 after retry RPC after error@1554106715) -> (continuation of Deferred@1327119269 after retry RPC after error@1353012525) -> (continuation of Deferred@1041201057 after retry RPC after error@1823902250) -> (continuation of Deferred@1928286986 after retry RPC after error@135887943)))
02:59:54.542 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [3ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@303172654(state=PENDING, result=null, callback=(continuation of Deferred@794910941 after retry RPC after error@2090247415) -> (continuation of Deferred@970160482 after retry RPC after error@992112906) -> (continuation of Deferred@534874427 after retry RPC after error@1444489004) -> (continuation of Deferred@1956863204 after retry RPC after error@1554106715) -> (continuation of Deferred@1327119269 after retry RPC after error@1353012525) -> (continuation of Deferred@1041201057 after retry RPC after error@1823902250) -> (continuation of Deferred@1928286986 after retry RPC after error@135887943), errback=(continuation of Deferred@794910941 after retry RPC after error@2090247415) -> (continuation of Deferred@970160482 after retry RPC after error@992112906) -> (continuation of Deferred@534874427 after retry RPC after error@1444489004) -> (continuation of Deferred@1956863204 after retry RPC after error@1554106715) -> (continuation of Deferred@1327119269 after retry RPC after error@1353012525) -> (continuation of Deferred@1041201057 after retry RPC after error@1823902250) -> (continuation of Deferred@1928286986 after retry RPC after error@135887943)))
02:59:54.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.561 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.561 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.561 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.561 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.561 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.561 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.561 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.562 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.579 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.580 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.580 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.580 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.580 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.580 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.580 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.599 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.600 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.600 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.600 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.600 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.600 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.600 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.621 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.621 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.621 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.621 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.621 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.621 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.621 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.622 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.641 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.641 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.641 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.641 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.641 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.641 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.642 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.661 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.661 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.661 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.661 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.661 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.661 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.662 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.699 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.700 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.700 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.700 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.700 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.700 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.700 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.780 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.780 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.781 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.781 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.781 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.781 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.782 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1715752954(state=PENDING, result=null, callback=(continuation of Deferred@2141886681 after retry RPC after error@368933002) -> (continuation of Deferred@2037499065 after retry RPC after error@1032749042) -> (continuation of Deferred@373900490 after retry RPC after error@1745766305) -> (continuation of Deferred@523043349 after retry RPC after error@1360170989) -> (continuation of Deferred@228716049 after retry RPC after error@386437963) -> (continuation of Deferred@1623332373 after retry RPC after error@636129287) -> (continuation of Deferred@658056141 after retry RPC after error@1954053802), errback=(continuation of Deferred@2141886681 after retry RPC after error@368933002) -> (continuation of Deferred@2037499065 after retry RPC after error@1032749042) -> (continuation of Deferred@373900490 after retry RPC after error@1745766305) -> (continuation of Deferred@523043349 after retry RPC after error@1360170989) -> (continuation of Deferred@228716049 after retry RPC after error@386437963) -> (continuation of Deferred@1623332373 after retry RPC after error@636129287) -> (continuation of Deferred@658056141 after retry RPC after error@1954053802)))
02:59:54.782 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1715752954(state=PENDING, result=null, callback=(continuation of Deferred@2141886681 after retry RPC after error@368933002) -> (continuation of Deferred@2037499065 after retry RPC after error@1032749042) -> (continuation of Deferred@373900490 after retry RPC after error@1745766305) -> (continuation of Deferred@523043349 after retry RPC after error@1360170989) -> (continuation of Deferred@228716049 after retry RPC after error@386437963) -> (continuation of Deferred@1623332373 after retry RPC after error@636129287) -> (continuation of Deferred@658056141 after retry RPC after error@1954053802), errback=(continuation of Deferred@2141886681 after retry RPC after error@368933002) -> (continuation of Deferred@2037499065 after retry RPC after error@1032749042) -> (continuation of Deferred@373900490 after retry RPC after error@1745766305) -> (continuation of Deferred@523043349 after retry RPC after error@1360170989) -> (continuation of Deferred@228716049 after retry RPC after error@386437963) -> (continuation of Deferred@1623332373 after retry RPC after error@636129287) -> (continuation of Deferred@658056141 after retry RPC after error@1954053802)))
02:59:54.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.801 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.801 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.801 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.801 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.801 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.801 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.820 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.820 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.821 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.821 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.821 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.821 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.841 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.841 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.841 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.841 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.841 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.841 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.861 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.861 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.861 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.861 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.861 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.861 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.881 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.881 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.881 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.881 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.881 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.881 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.882 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.901 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.901 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.901 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.901 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.901 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.901 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.902 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.979 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:54.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:54.980 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.980 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.980 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:54.980 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.980 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.980 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:54.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:54.999 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.000 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.000 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.000 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.000 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.000 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.000 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.039 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.040 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.040 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.040 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.040 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.040 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.040 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.041 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.041 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@2103274405(state=PENDING, result=null, callback=(continuation of Deferred@54001326 after retry RPC after error@1486884448) -> (continuation of Deferred@1634230277 after retry RPC after error@2013147086) -> (continuation of Deferred@1258323217 after retry RPC after error@319535391) -> (continuation of Deferred@826383155 after retry RPC after error@935805578) -> (continuation of Deferred@2145868182 after retry RPC after error@829080628) -> (continuation of Deferred@214911273 after retry RPC after error@17930536) -> (continuation of Deferred@1157581685 after retry RPC after error@592669882) -> (continuation of Deferred@1428422648 after retry RPC after error@1492579501), errback=(continuation of Deferred@54001326 after retry RPC after error@1486884448) -> (continuation of Deferred@1634230277 after retry RPC after error@2013147086) -> (continuation of Deferred@1258323217 after retry RPC after error@319535391) -> (continuation of Deferred@826383155 after retry RPC after error@935805578) -> (continuation of Deferred@2145868182 after retry RPC after error@829080628) -> (continuation of Deferred@214911273 after retry RPC after error@17930536) -> (continuation of Deferred@1157581685 after retry RPC after error@592669882) -> (continuation of Deferred@1428422648 after retry RPC after error@1492579501)))
02:59:55.042 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@2103274405(state=PENDING, result=null, callback=(continuation of Deferred@54001326 after retry RPC after error@1486884448) -> (continuation of Deferred@1634230277 after retry RPC after error@2013147086) -> (continuation of Deferred@1258323217 after retry RPC after error@319535391) -> (continuation of Deferred@826383155 after retry RPC after error@935805578) -> (continuation of Deferred@2145868182 after retry RPC after error@829080628) -> (continuation of Deferred@214911273 after retry RPC after error@17930536) -> (continuation of Deferred@1157581685 after retry RPC after error@592669882) -> (continuation of Deferred@1428422648 after retry RPC after error@1492579501), errback=(continuation of Deferred@54001326 after retry RPC after error@1486884448) -> (continuation of Deferred@1634230277 after retry RPC after error@2013147086) -> (continuation of Deferred@1258323217 after retry RPC after error@319535391) -> (continuation of Deferred@826383155 after retry RPC after error@935805578) -> (continuation of Deferred@2145868182 after retry RPC after error@829080628) -> (continuation of Deferred@214911273 after retry RPC after error@17930536) -> (continuation of Deferred@1157581685 after retry RPC after error@592669882) -> (continuation of Deferred@1428422648 after retry RPC after error@1492579501)))
02:59:55.059 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.060 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.060 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.060 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.060 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.060 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.060 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.062 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.080 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.080 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.080 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.080 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.080 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.081 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.081 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.099 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.100 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.100 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.100 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.100 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.100 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.100 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.119 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.120 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.120 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.120 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.120 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.120 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.120 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.139 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.140 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.140 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.140 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.140 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.140 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.140 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.141 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.159 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.160 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.160 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.160 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.160 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.160 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.160 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.199 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.200 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.200 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.200 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.200 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.200 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.200 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.259 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.260 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.260 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.260 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.260 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.260 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.260 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.261 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.261 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2019998102(state=PENDING, result=null, callback=(continuation of Deferred@926739898 after retry RPC after error@97375814) -> (continuation of Deferred@1046781822 after retry RPC after error@1929591597) -> (continuation of Deferred@595741130 after retry RPC after error@1217844250) -> (continuation of Deferred@1376988441 after retry RPC after error@941862437) -> (continuation of Deferred@165571826 after retry RPC after error@1997950935) -> (continuation of Deferred@1017569225 after retry RPC after error@424746238) -> (continuation of Deferred@1670535898 after retry RPC after error@641714568), errback=(continuation of Deferred@926739898 after retry RPC after error@97375814) -> (continuation of Deferred@1046781822 after retry RPC after error@1929591597) -> (continuation of Deferred@595741130 after retry RPC after error@1217844250) -> (continuation of Deferred@1376988441 after retry RPC after error@941862437) -> (continuation of Deferred@165571826 after retry RPC after error@1997950935) -> (continuation of Deferred@1017569225 after retry RPC after error@424746238) -> (continuation of Deferred@1670535898 after retry RPC after error@641714568)))
02:59:55.262 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2019998102(state=PENDING, result=null, callback=(continuation of Deferred@926739898 after retry RPC after error@97375814) -> (continuation of Deferred@1046781822 after retry RPC after error@1929591597) -> (continuation of Deferred@595741130 after retry RPC after error@1217844250) -> (continuation of Deferred@1376988441 after retry RPC after error@941862437) -> (continuation of Deferred@165571826 after retry RPC after error@1997950935) -> (continuation of Deferred@1017569225 after retry RPC after error@424746238) -> (continuation of Deferred@1670535898 after retry RPC after error@641714568), errback=(continuation of Deferred@926739898 after retry RPC after error@97375814) -> (continuation of Deferred@1046781822 after retry RPC after error@1929591597) -> (continuation of Deferred@595741130 after retry RPC after error@1217844250) -> (continuation of Deferred@1376988441 after retry RPC after error@941862437) -> (continuation of Deferred@165571826 after retry RPC after error@1997950935) -> (continuation of Deferred@1017569225 after retry RPC after error@424746238) -> (continuation of Deferred@1670535898 after retry RPC after error@641714568)))
02:59:55.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:55.268069 21270 raft_consensus.cc:534] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 56099a6022da440a81f20e6fe8903fb0)
02:59:55.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:55.268347 21270 raft_consensus.cc:556] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
02:59:55.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:55.269383 21270 leader_election.cc:290] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
02:59:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:55.271771 20378 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111): Network error: Client connection negotiation failed: client connection to 127.19.203.254:46111: connect: Connection refused (error 111)
02:59:55.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 02:59:55.271981 20381 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
02:59:55.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:55.272110 20381 leader_election.cc:304] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c; no voters: 56099a6022da440a81f20e6fe8903fb0, efb4c69b20bd4f9692d496c33ddbb24e
02:59:55.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 02:59:55.272357 21270 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:59:55.279 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.280 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.280 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.280 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.280 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.280 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.280 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.299 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.300 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.300 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.300 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.300 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.300 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.300 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.321 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.321 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.321 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.321 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.321 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.321 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.321 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.322 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.341 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.341 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.341 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.341 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.341 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.341 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.342 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.361 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.361 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.361 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.361 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.361 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.361 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.401 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.401 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.401 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.401 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.401 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.401 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.421 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.421 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.421 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.421 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.421 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.421 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.422 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.519 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.520 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.520 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.520 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.520 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.520 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.520 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.521 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1349354232(state=PENDING, result=null, callback=(continuation of Deferred@1059361750 after retry RPC after error@256406633) -> (continuation of Deferred@949268950 after retry RPC after error@80911998) -> (continuation of Deferred@536024849 after retry RPC after error@1398064058) -> (continuation of Deferred@1314116103 after retry RPC after error@1583518941) -> (continuation of Deferred@1504594847 after retry RPC after error@739379715) -> (continuation of Deferred@2080450156 after retry RPC after error@915374413) -> (continuation of Deferred@500618928 after retry RPC after error@556303643), errback=(continuation of Deferred@1059361750 after retry RPC after error@256406633) -> (continuation of Deferred@949268950 after retry RPC after error@80911998) -> (continuation of Deferred@536024849 after retry RPC after error@1398064058) -> (continuation of Deferred@1314116103 after retry RPC after error@1583518941) -> (continuation of Deferred@1504594847 after retry RPC after error@739379715) -> (continuation of Deferred@2080450156 after retry RPC after error@915374413) -> (continuation of Deferred@500618928 after retry RPC after error@556303643)))
02:59:55.522 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1349354232(state=PENDING, result=null, callback=(continuation of Deferred@1059361750 after retry RPC after error@256406633) -> (continuation of Deferred@949268950 after retry RPC after error@80911998) -> (continuation of Deferred@536024849 after retry RPC after error@1398064058) -> (continuation of Deferred@1314116103 after retry RPC after error@1583518941) -> (continuation of Deferred@1504594847 after retry RPC after error@739379715) -> (continuation of Deferred@2080450156 after retry RPC after error@915374413) -> (continuation of Deferred@500618928 after retry RPC after error@556303643), errback=(continuation of Deferred@1059361750 after retry RPC after error@256406633) -> (continuation of Deferred@949268950 after retry RPC after error@80911998) -> (continuation of Deferred@536024849 after retry RPC after error@1398064058) -> (continuation of Deferred@1314116103 after retry RPC after error@1583518941) -> (continuation of Deferred@1504594847 after retry RPC after error@739379715) -> (continuation of Deferred@2080450156 after retry RPC after error@915374413) -> (continuation of Deferred@500618928 after retry RPC after error@556303643)))
02:59:55.539 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.540 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.540 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.540 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.540 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.540 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.540 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.559 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.560 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.560 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.560 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.560 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.561 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.561 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.561 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.581 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.581 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.581 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.581 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.581 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.581 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.581 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.582 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.601 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.601 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.601 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.601 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.601 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.601 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.601 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.602 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.619 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.620 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.620 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.620 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.620 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.621 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.621 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.659 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.660 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.660 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.660 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.660 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.660 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.660 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.720 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.720 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.720 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.720 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.721 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.721 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.721 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.721 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@2069980852(state=PENDING, result=null, callback=(continuation of Deferred@1631726366 after retry RPC after error@292698740) -> (continuation of Deferred@783752432 after retry RPC after error@1578343252) -> (continuation of Deferred@779702611 after retry RPC after error@777547634) -> (continuation of Deferred@941536981 after retry RPC after error@2048218041) -> (continuation of Deferred@503473761 after retry RPC after error@1934403884) -> (continuation of Deferred@1051263374 after retry RPC after error@1861289337), errback=(continuation of Deferred@1631726366 after retry RPC after error@292698740) -> (continuation of Deferred@783752432 after retry RPC after error@1578343252) -> (continuation of Deferred@779702611 after retry RPC after error@777547634) -> (continuation of Deferred@941536981 after retry RPC after error@2048218041) -> (continuation of Deferred@503473761 after retry RPC after error@1934403884) -> (continuation of Deferred@1051263374 after retry RPC after error@1861289337)))
02:59:55.722 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@2069980852(state=PENDING, result=null, callback=(continuation of Deferred@1631726366 after retry RPC after error@292698740) -> (continuation of Deferred@783752432 after retry RPC after error@1578343252) -> (continuation of Deferred@779702611 after retry RPC after error@777547634) -> (continuation of Deferred@941536981 after retry RPC after error@2048218041) -> (continuation of Deferred@503473761 after retry RPC after error@1934403884) -> (continuation of Deferred@1051263374 after retry RPC after error@1861289337), errback=(continuation of Deferred@1631726366 after retry RPC after error@292698740) -> (continuation of Deferred@783752432 after retry RPC after error@1578343252) -> (continuation of Deferred@779702611 after retry RPC after error@777547634) -> (continuation of Deferred@941536981 after retry RPC after error@2048218041) -> (continuation of Deferred@503473761 after retry RPC after error@1934403884) -> (continuation of Deferred@1051263374 after retry RPC after error@1861289337)))
02:59:55.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.740 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.740 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.740 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.740 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.740 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.741 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.759 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.760 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.760 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.760 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.760 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.760 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.760 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.780 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.780 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.780 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.781 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.781 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.781 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.801 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.801 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.801 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.801 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.801 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.801 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.821 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.821 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.821 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.821 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.821 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.821 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.822 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.840 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.840 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.841 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.841 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.841 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.841 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.842 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.860 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.861 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.861 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.861 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.861 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.861 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:55.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:55.980 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.980 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.980 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:55.981 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.981 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.981 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:55.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:55.982 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1825092012(state=PENDING, result=null, callback=(continuation of Deferred@1840180406 after retry RPC after error@69327422) -> (continuation of Deferred@1806359661 after retry RPC after error@1111901541) -> (continuation of Deferred@1486342855 after retry RPC after error@1391266753) -> (continuation of Deferred@1872362565 after retry RPC after error@520417726) -> (continuation of Deferred@2040779115 after retry RPC after error@1126858397) -> (continuation of Deferred@1676790121 after retry RPC after error@1539329448) -> (continuation of Deferred@1007679282 after retry RPC after error@742504632), errback=(continuation of Deferred@1840180406 after retry RPC after error@69327422) -> (continuation of Deferred@1806359661 after retry RPC after error@1111901541) -> (continuation of Deferred@1486342855 after retry RPC after error@1391266753) -> (continuation of Deferred@1872362565 after retry RPC after error@520417726) -> (continuation of Deferred@2040779115 after retry RPC after error@1126858397) -> (continuation of Deferred@1676790121 after retry RPC after error@1539329448) -> (continuation of Deferred@1007679282 after retry RPC after error@742504632)))
02:59:55.982 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1825092012(state=PENDING, result=null, callback=(continuation of Deferred@1840180406 after retry RPC after error@69327422) -> (continuation of Deferred@1806359661 after retry RPC after error@1111901541) -> (continuation of Deferred@1486342855 after retry RPC after error@1391266753) -> (continuation of Deferred@1872362565 after retry RPC after error@520417726) -> (continuation of Deferred@2040779115 after retry RPC after error@1126858397) -> (continuation of Deferred@1676790121 after retry RPC after error@1539329448) -> (continuation of Deferred@1007679282 after retry RPC after error@742504632), errback=(continuation of Deferred@1840180406 after retry RPC after error@69327422) -> (continuation of Deferred@1806359661 after retry RPC after error@1111901541) -> (continuation of Deferred@1486342855 after retry RPC after error@1391266753) -> (continuation of Deferred@1872362565 after retry RPC after error@520417726) -> (continuation of Deferred@2040779115 after retry RPC after error@1126858397) -> (continuation of Deferred@1676790121 after retry RPC after error@1539329448) -> (continuation of Deferred@1007679282 after retry RPC after error@742504632)))
02:59:56.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.000 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.000 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.000 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.000 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.001 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.001 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.002 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.020 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.020 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.020 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.020 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.020 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.020 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.039 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.040 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.040 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.040 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.040 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.040 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.040 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.041 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.059 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.060 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.060 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.060 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.060 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.061 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.061 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.061 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.079 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.080 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.080 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.080 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.080 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.080 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.080 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.081 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.099 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.100 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.100 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.100 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.100 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.100 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.100 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.100 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.100 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.160 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.160 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.161 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.161 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.161 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.161 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.221 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.221 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.221 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.221 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.221 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.221 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.222 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@266855020(state=PENDING, result=null, callback=(continuation of Deferred@117351689 after retry RPC after error@1659706908) -> (continuation of Deferred@141084238 after retry RPC after error@942449069) -> (continuation of Deferred@1258921292 after retry RPC after error@1092511695) -> (continuation of Deferred@260739860 after retry RPC after error@1391884929) -> (continuation of Deferred@1435659886 after retry RPC after error@1736461068) -> (continuation of Deferred@1019836123 after retry RPC after error@1560949169) -> (continuation of Deferred@464373088 after retry RPC after error@2123380018), errback=(continuation of Deferred@117351689 after retry RPC after error@1659706908) -> (continuation of Deferred@141084238 after retry RPC after error@942449069) -> (continuation of Deferred@1258921292 after retry RPC after error@1092511695) -> (continuation of Deferred@260739860 after retry RPC after error@1391884929) -> (continuation of Deferred@1435659886 after retry RPC after error@1736461068) -> (continuation of Deferred@1019836123 after retry RPC after error@1560949169) -> (continuation of Deferred@464373088 after retry RPC after error@2123380018)))
02:59:56.222 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@266855020(state=PENDING, result=null, callback=(continuation of Deferred@117351689 after retry RPC after error@1659706908) -> (continuation of Deferred@141084238 after retry RPC after error@942449069) -> (continuation of Deferred@1258921292 after retry RPC after error@1092511695) -> (continuation of Deferred@260739860 after retry RPC after error@1391884929) -> (continuation of Deferred@1435659886 after retry RPC after error@1736461068) -> (continuation of Deferred@1019836123 after retry RPC after error@1560949169) -> (continuation of Deferred@464373088 after retry RPC after error@2123380018), errback=(continuation of Deferred@117351689 after retry RPC after error@1659706908) -> (continuation of Deferred@141084238 after retry RPC after error@942449069) -> (continuation of Deferred@1258921292 after retry RPC after error@1092511695) -> (continuation of Deferred@260739860 after retry RPC after error@1391884929) -> (continuation of Deferred@1435659886 after retry RPC after error@1736461068) -> (continuation of Deferred@1019836123 after retry RPC after error@1560949169) -> (continuation of Deferred@464373088 after retry RPC after error@2123380018)))
02:59:56.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.241 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.241 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.241 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.241 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.241 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.241 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.242 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.261 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.261 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.261 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.261 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.261 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.261 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.262 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.281 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.281 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.281 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.281 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.281 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.281 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.281 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.281 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.282 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.301 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.301 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.301 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.301 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.301 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.301 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.320 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.320 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.320 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.321 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.321 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.321 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.340 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.340 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.340 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.340 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.340 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.340 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.400 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.401 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.401 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.401 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.401 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.401 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.402 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.441 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.441 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.441 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.441 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.441 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.441 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.442 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.442 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1093436176(state=PENDING, result=null, callback=(continuation of Deferred@1510021567 after retry RPC after error@774406297) -> (continuation of Deferred@295210987 after retry RPC after error@873052875) -> (continuation of Deferred@1721205012 after retry RPC after error@215087383) -> (continuation of Deferred@1508104369 after retry RPC after error@549842596) -> (continuation of Deferred@2046837897 after retry RPC after error@2031747999) -> (continuation of Deferred@1048275549 after retry RPC after error@826541048) -> (continuation of Deferred@1234709845 after retry RPC after error@1622657282), errback=(continuation of Deferred@1510021567 after retry RPC after error@774406297) -> (continuation of Deferred@295210987 after retry RPC after error@873052875) -> (continuation of Deferred@1721205012 after retry RPC after error@215087383) -> (continuation of Deferred@1508104369 after retry RPC after error@549842596) -> (continuation of Deferred@2046837897 after retry RPC after error@2031747999) -> (continuation of Deferred@1048275549 after retry RPC after error@826541048) -> (continuation of Deferred@1234709845 after retry RPC after error@1622657282)))
02:59:56.442 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1093436176(state=PENDING, result=null, callback=(continuation of Deferred@1510021567 after retry RPC after error@774406297) -> (continuation of Deferred@295210987 after retry RPC after error@873052875) -> (continuation of Deferred@1721205012 after retry RPC after error@215087383) -> (continuation of Deferred@1508104369 after retry RPC after error@549842596) -> (continuation of Deferred@2046837897 after retry RPC after error@2031747999) -> (continuation of Deferred@1048275549 after retry RPC after error@826541048) -> (continuation of Deferred@1234709845 after retry RPC after error@1622657282), errback=(continuation of Deferred@1510021567 after retry RPC after error@774406297) -> (continuation of Deferred@295210987 after retry RPC after error@873052875) -> (continuation of Deferred@1721205012 after retry RPC after error@215087383) -> (continuation of Deferred@1508104369 after retry RPC after error@549842596) -> (continuation of Deferred@2046837897 after retry RPC after error@2031747999) -> (continuation of Deferred@1048275549 after retry RPC after error@826541048) -> (continuation of Deferred@1234709845 after retry RPC after error@1622657282)))
02:59:56.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.461 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.461 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.461 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.461 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.461 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.461 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.462 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.481 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.481 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.481 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.481 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.481 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.481 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.482 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.501 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.501 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.501 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.501 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.501 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.501 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.519 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.520 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.520 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.520 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.520 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.520 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.520 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.541 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.541 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.541 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.541 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.541 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.541 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.541 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.542 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.561 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.561 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.561 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.561 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.561 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.561 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.562 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.581 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.581 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.581 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.581 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.581 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.581 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.582 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.601 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.601 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.601 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.601 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.601 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.601 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.602 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.602 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@481924587(state=PENDING, result=null, callback=(continuation of Deferred@127994813 after retry RPC after error@1202852763) -> (continuation of Deferred@1358806344 after retry RPC after error@12287089) -> (continuation of Deferred@264794936 after retry RPC after error@1479645073) -> (continuation of Deferred@1696294434 after retry RPC after error@1041815044) -> (continuation of Deferred@1912939340 after retry RPC after error@651997058) -> (continuation of Deferred@415819533 after retry RPC after error@1612229350) -> (continuation of Deferred@1004333802 after retry RPC after error@2017456812), errback=(continuation of Deferred@127994813 after retry RPC after error@1202852763) -> (continuation of Deferred@1358806344 after retry RPC after error@12287089) -> (continuation of Deferred@264794936 after retry RPC after error@1479645073) -> (continuation of Deferred@1696294434 after retry RPC after error@1041815044) -> (continuation of Deferred@1912939340 after retry RPC after error@651997058) -> (continuation of Deferred@415819533 after retry RPC after error@1612229350) -> (continuation of Deferred@1004333802 after retry RPC after error@2017456812)))
02:59:56.603 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@481924587(state=PENDING, result=null, callback=(continuation of Deferred@127994813 after retry RPC after error@1202852763) -> (continuation of Deferred@1358806344 after retry RPC after error@12287089) -> (continuation of Deferred@264794936 after retry RPC after error@1479645073) -> (continuation of Deferred@1696294434 after retry RPC after error@1041815044) -> (continuation of Deferred@1912939340 after retry RPC after error@651997058) -> (continuation of Deferred@415819533 after retry RPC after error@1612229350) -> (continuation of Deferred@1004333802 after retry RPC after error@2017456812), errback=(continuation of Deferred@127994813 after retry RPC after error@1202852763) -> (continuation of Deferred@1358806344 after retry RPC after error@12287089) -> (continuation of Deferred@264794936 after retry RPC after error@1479645073) -> (continuation of Deferred@1696294434 after retry RPC after error@1041815044) -> (continuation of Deferred@1912939340 after retry RPC after error@651997058) -> (continuation of Deferred@415819533 after retry RPC after error@1612229350) -> (continuation of Deferred@1004333802 after retry RPC after error@2017456812)))
02:59:56.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.621 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.621 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.621 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.621 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.621 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.621 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.641 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.641 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.641 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.641 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.641 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.641 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.661 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.661 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.661 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.661 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.661 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.661 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.681 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.681 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.681 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.681 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.681 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.681 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.682 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.701 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.701 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.701 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.701 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.701 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.701 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.702 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.739 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.740 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.740 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.740 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.740 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.740 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.740 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.761 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.761 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.761 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.761 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.761 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.761 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.762 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.781 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.781 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.781 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.781 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.782 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.782 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.782 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.782 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.800 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.800 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.801 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.801 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.801 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.801 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.802 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2049081744(state=PENDING, result=null, callback=(continuation of Deferred@706019992 after retry RPC after error@521669968) -> (continuation of Deferred@1145658665 after retry RPC after error@713652785) -> (continuation of Deferred@680176698 after retry RPC after error@2007451382) -> (continuation of Deferred@2030693694 after retry RPC after error@414210268) -> (continuation of Deferred@1705430156 after retry RPC after error@687584824) -> (continuation of Deferred@506318340 after retry RPC after error@79762341) -> (continuation of Deferred@1473281309 after retry RPC after error@577723305) -> (continuation of Deferred@1388506126 after retry RPC after error@1586663880), errback=(continuation of Deferred@706019992 after retry RPC after error@521669968) -> (continuation of Deferred@1145658665 after retry RPC after error@713652785) -> (continuation of Deferred@680176698 after retry RPC after error@2007451382) -> (continuation of Deferred@2030693694 after retry RPC after error@414210268) -> (continuation of Deferred@1705430156 after retry RPC after error@687584824) -> (continuation of Deferred@506318340 after retry RPC after error@79762341) -> (continuation of Deferred@1473281309 after retry RPC after error@577723305) -> (continuation of Deferred@1388506126 after retry RPC after error@1586663880)))
02:59:56.802 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2049081744(state=PENDING, result=null, callback=(continuation of Deferred@706019992 after retry RPC after error@521669968) -> (continuation of Deferred@1145658665 after retry RPC after error@713652785) -> (continuation of Deferred@680176698 after retry RPC after error@2007451382) -> (continuation of Deferred@2030693694 after retry RPC after error@414210268) -> (continuation of Deferred@1705430156 after retry RPC after error@687584824) -> (continuation of Deferred@506318340 after retry RPC after error@79762341) -> (continuation of Deferred@1473281309 after retry RPC after error@577723305) -> (continuation of Deferred@1388506126 after retry RPC after error@1586663880), errback=(continuation of Deferred@706019992 after retry RPC after error@521669968) -> (continuation of Deferred@1145658665 after retry RPC after error@713652785) -> (continuation of Deferred@680176698 after retry RPC after error@2007451382) -> (continuation of Deferred@2030693694 after retry RPC after error@414210268) -> (continuation of Deferred@1705430156 after retry RPC after error@687584824) -> (continuation of Deferred@506318340 after retry RPC after error@79762341) -> (continuation of Deferred@1473281309 after retry RPC after error@577723305) -> (continuation of Deferred@1388506126 after retry RPC after error@1586663880)))
02:59:56.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.820 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.820 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.820 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.820 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.820 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.821 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.839 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.840 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.840 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.840 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.840 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.840 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.840 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.861 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.861 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.861 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.861 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.861 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.861 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.861 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.862 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.879 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.880 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.880 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.880 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.880 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.880 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.880 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.901 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.901 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.901 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.901 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.901 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.901 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.921 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.921 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.921 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.921 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.921 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.921 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:56.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:56.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:56.961 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.961 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.961 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.961 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.961 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:56.961 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:56.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.059 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.060 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.060 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.060 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.060 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.060 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.060 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.061 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1346733658(state=PENDING, result=null, callback=(continuation of Deferred@1512171902 after retry RPC after error@747686294) -> (continuation of Deferred@1792859231 after retry RPC after error@383250440) -> (continuation of Deferred@1130922421 after retry RPC after error@1771977783) -> (continuation of Deferred@1697397631 after retry RPC after error@1288884322) -> (continuation of Deferred@1261471076 after retry RPC after error@1398241201) -> (continuation of Deferred@1470384487 after retry RPC after error@229626417) -> (continuation of Deferred@1115385157 after retry RPC after error@712597572), errback=(continuation of Deferred@1512171902 after retry RPC after error@747686294) -> (continuation of Deferred@1792859231 after retry RPC after error@383250440) -> (continuation of Deferred@1130922421 after retry RPC after error@1771977783) -> (continuation of Deferred@1697397631 after retry RPC after error@1288884322) -> (continuation of Deferred@1261471076 after retry RPC after error@1398241201) -> (continuation of Deferred@1470384487 after retry RPC after error@229626417) -> (continuation of Deferred@1115385157 after retry RPC after error@712597572)))
02:59:57.062 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1346733658(state=PENDING, result=null, callback=(continuation of Deferred@1512171902 after retry RPC after error@747686294) -> (continuation of Deferred@1792859231 after retry RPC after error@383250440) -> (continuation of Deferred@1130922421 after retry RPC after error@1771977783) -> (continuation of Deferred@1697397631 after retry RPC after error@1288884322) -> (continuation of Deferred@1261471076 after retry RPC after error@1398241201) -> (continuation of Deferred@1470384487 after retry RPC after error@229626417) -> (continuation of Deferred@1115385157 after retry RPC after error@712597572), errback=(continuation of Deferred@1512171902 after retry RPC after error@747686294) -> (continuation of Deferred@1792859231 after retry RPC after error@383250440) -> (continuation of Deferred@1130922421 after retry RPC after error@1771977783) -> (continuation of Deferred@1697397631 after retry RPC after error@1288884322) -> (continuation of Deferred@1261471076 after retry RPC after error@1398241201) -> (continuation of Deferred@1470384487 after retry RPC after error@229626417) -> (continuation of Deferred@1115385157 after retry RPC after error@712597572)))
02:59:57.079 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.080 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.080 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.080 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.080 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.080 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.080 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.081 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.099 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.100 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.100 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.100 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.100 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.100 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.100 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.119 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.120 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.120 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.120 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.120 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.120 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.120 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.141 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.141 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.141 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.141 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.142 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.142 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.142 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.142 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.161 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.161 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.161 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.161 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.161 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.161 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.181 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.181 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.181 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.181 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.181 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.181 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.261 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.261 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.261 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.261 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.262 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.262 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.262 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.262 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.262 [DEBUG - kudu-nio-4] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1891966801(state=PENDING, result=null, callback=(continuation of Deferred@1020279348 after retry RPC after error@692012193) -> (continuation of Deferred@1754078873 after retry RPC after error@820685646) -> (continuation of Deferred@232138493 after retry RPC after error@1579114922) -> (continuation of Deferred@1592182054 after retry RPC after error@1001633135) -> (continuation of Deferred@124689706 after retry RPC after error@584695388) -> (continuation of Deferred@1258197204 after retry RPC after error@1816153129), errback=(continuation of Deferred@1020279348 after retry RPC after error@692012193) -> (continuation of Deferred@1754078873 after retry RPC after error@820685646) -> (continuation of Deferred@232138493 after retry RPC after error@1579114922) -> (continuation of Deferred@1592182054 after retry RPC after error@1001633135) -> (continuation of Deferred@124689706 after retry RPC after error@584695388) -> (continuation of Deferred@1258197204 after retry RPC after error@1816153129)))
02:59:57.263 [DEBUG - kudu-nio-4] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1891966801(state=PENDING, result=null, callback=(continuation of Deferred@1020279348 after retry RPC after error@692012193) -> (continuation of Deferred@1754078873 after retry RPC after error@820685646) -> (continuation of Deferred@232138493 after retry RPC after error@1579114922) -> (continuation of Deferred@1592182054 after retry RPC after error@1001633135) -> (continuation of Deferred@124689706 after retry RPC after error@584695388) -> (continuation of Deferred@1258197204 after retry RPC after error@1816153129), errback=(continuation of Deferred@1020279348 after retry RPC after error@692012193) -> (continuation of Deferred@1754078873 after retry RPC after error@820685646) -> (continuation of Deferred@232138493 after retry RPC after error@1579114922) -> (continuation of Deferred@1592182054 after retry RPC after error@1001633135) -> (continuation of Deferred@124689706 after retry RPC after error@584695388) -> (continuation of Deferred@1258197204 after retry RPC after error@1816153129)))
02:59:57.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.281 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.281 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.281 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.281 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.281 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.281 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.300 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.300 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.301 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.301 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.301 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.301 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.320 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.320 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.320 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.320 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.320 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.320 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.339 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.340 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.340 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.340 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.340 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.340 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.340 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.361 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.361 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.361 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.361 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.361 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.361 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.401 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.401 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.401 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.401 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.401 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.401 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.402 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.439 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.440 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.440 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.440 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.440 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.441 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.441 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.441 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.479 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.480 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.480 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.480 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.480 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.480 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.480 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.481 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.481 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1458450378(state=PENDING, result=null, callback=(continuation of Deferred@841519585 after retry RPC after error@945312088) -> (continuation of Deferred@1384439027 after retry RPC after error@1966291222) -> (continuation of Deferred@65145288 after retry RPC after error@2043300477) -> (continuation of Deferred@232616691 after retry RPC after error@1640632791) -> (continuation of Deferred@1920333935 after retry RPC after error@1497878014) -> (continuation of Deferred@1064007375 after retry RPC after error@236883036) -> (continuation of Deferred@1400220528 after retry RPC after error@884651229), errback=(continuation of Deferred@841519585 after retry RPC after error@945312088) -> (continuation of Deferred@1384439027 after retry RPC after error@1966291222) -> (continuation of Deferred@65145288 after retry RPC after error@2043300477) -> (continuation of Deferred@232616691 after retry RPC after error@1640632791) -> (continuation of Deferred@1920333935 after retry RPC after error@1497878014) -> (continuation of Deferred@1064007375 after retry RPC after error@236883036) -> (continuation of Deferred@1400220528 after retry RPC after error@884651229)))
02:59:57.481 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1458450378(state=PENDING, result=null, callback=(continuation of Deferred@841519585 after retry RPC after error@945312088) -> (continuation of Deferred@1384439027 after retry RPC after error@1966291222) -> (continuation of Deferred@65145288 after retry RPC after error@2043300477) -> (continuation of Deferred@232616691 after retry RPC after error@1640632791) -> (continuation of Deferred@1920333935 after retry RPC after error@1497878014) -> (continuation of Deferred@1064007375 after retry RPC after error@236883036) -> (continuation of Deferred@1400220528 after retry RPC after error@884651229), errback=(continuation of Deferred@841519585 after retry RPC after error@945312088) -> (continuation of Deferred@1384439027 after retry RPC after error@1966291222) -> (continuation of Deferred@65145288 after retry RPC after error@2043300477) -> (continuation of Deferred@232616691 after retry RPC after error@1640632791) -> (continuation of Deferred@1920333935 after retry RPC after error@1497878014) -> (continuation of Deferred@1064007375 after retry RPC after error@236883036) -> (continuation of Deferred@1400220528 after retry RPC after error@884651229)))
02:59:57.499 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.500 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.500 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.500 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.500 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.500 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.500 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.519 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.520 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.520 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.520 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.521 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.521 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.521 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.541 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.541 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.541 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.541 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.541 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.541 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.560 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.560 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.560 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.561 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.561 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.561 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.561 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.580 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.580 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.580 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.580 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.581 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.581 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.599 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.600 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.600 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.600 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.601 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.601 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.601 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.639 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.640 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.640 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.640 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.640 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.640 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.640 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.701 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.701 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.701 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.701 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.701 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.701 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.702 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2121007532(state=PENDING, result=null, callback=(continuation of Deferred@2142631417 after retry RPC after error@12749252) -> (continuation of Deferred@1955812638 after retry RPC after error@1785486145) -> (continuation of Deferred@975593928 after retry RPC after error@775836641) -> (continuation of Deferred@1089194141 after retry RPC after error@1221932434) -> (continuation of Deferred@1098134881 after retry RPC after error@487184373) -> (continuation of Deferred@54025193 after retry RPC after error@1654691769) -> (continuation of Deferred@35617201 after retry RPC after error@1375383672), errback=(continuation of Deferred@2142631417 after retry RPC after error@12749252) -> (continuation of Deferred@1955812638 after retry RPC after error@1785486145) -> (continuation of Deferred@975593928 after retry RPC after error@775836641) -> (continuation of Deferred@1089194141 after retry RPC after error@1221932434) -> (continuation of Deferred@1098134881 after retry RPC after error@487184373) -> (continuation of Deferred@54025193 after retry RPC after error@1654691769) -> (continuation of Deferred@35617201 after retry RPC after error@1375383672)))
02:59:57.702 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2121007532(state=PENDING, result=null, callback=(continuation of Deferred@2142631417 after retry RPC after error@12749252) -> (continuation of Deferred@1955812638 after retry RPC after error@1785486145) -> (continuation of Deferred@975593928 after retry RPC after error@775836641) -> (continuation of Deferred@1089194141 after retry RPC after error@1221932434) -> (continuation of Deferred@1098134881 after retry RPC after error@487184373) -> (continuation of Deferred@54025193 after retry RPC after error@1654691769) -> (continuation of Deferred@35617201 after retry RPC after error@1375383672), errback=(continuation of Deferred@2142631417 after retry RPC after error@12749252) -> (continuation of Deferred@1955812638 after retry RPC after error@1785486145) -> (continuation of Deferred@975593928 after retry RPC after error@775836641) -> (continuation of Deferred@1089194141 after retry RPC after error@1221932434) -> (continuation of Deferred@1098134881 after retry RPC after error@487184373) -> (continuation of Deferred@54025193 after retry RPC after error@1654691769) -> (continuation of Deferred@35617201 after retry RPC after error@1375383672)))
02:59:57.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.721 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.721 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.721 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.721 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.721 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.721 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.721 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.721 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.722 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.722 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.741 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.741 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.741 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.741 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.741 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.741 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.760 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.760 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.760 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.760 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.760 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.761 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.780 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.780 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.780 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.780 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.780 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.781 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.799 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.800 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.800 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.800 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.800 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.800 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.800 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.821 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.821 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.821 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.821 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.821 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.821 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.822 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.881 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.881 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.881 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.881 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.881 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.881 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.881 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.882 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.960 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.960 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.961 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.961 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.961 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.961 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:57.962 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=240), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [0ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [0ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [20ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [39ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [40ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [58ms] refreshing cache from master, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [59ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [60ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [78ms] refreshing cache from master, [78ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [78ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [79ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [79ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [80ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [238ms] refreshing cache from master, [238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [240ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2032842072(state=PENDING, result=null, callback=(continuation of Deferred@925987235 after retry RPC after error@1506670098) -> (continuation of Deferred@1533478597 after retry RPC after error@2001463428) -> (continuation of Deferred@1271608231 after retry RPC after error@2105549226) -> (continuation of Deferred@506614257 after retry RPC after error@907380091) -> (continuation of Deferred@609340949 after retry RPC after error@1486890564) -> (continuation of Deferred@1410423160 after retry RPC after error@1312104668) -> (continuation of Deferred@1681817193 after retry RPC after error@1031141036), errback=(continuation of Deferred@925987235 after retry RPC after error@1506670098) -> (continuation of Deferred@1533478597 after retry RPC after error@2001463428) -> (continuation of Deferred@1271608231 after retry RPC after error@2105549226) -> (continuation of Deferred@506614257 after retry RPC after error@907380091) -> (continuation of Deferred@609340949 after retry RPC after error@1486890564) -> (continuation of Deferred@1410423160 after retry RPC after error@1312104668) -> (continuation of Deferred@1681817193 after retry RPC after error@1031141036)))
02:59:57.962 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=240), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [0ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [0ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [20ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [39ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [40ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [58ms] refreshing cache from master, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [59ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [60ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [78ms] refreshing cache from master, [78ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [78ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [79ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [79ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [80ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [238ms] refreshing cache from master, [238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [240ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2032842072(state=PENDING, result=null, callback=(continuation of Deferred@925987235 after retry RPC after error@1506670098) -> (continuation of Deferred@1533478597 after retry RPC after error@2001463428) -> (continuation of Deferred@1271608231 after retry RPC after error@2105549226) -> (continuation of Deferred@506614257 after retry RPC after error@907380091) -> (continuation of Deferred@609340949 after retry RPC after error@1486890564) -> (continuation of Deferred@1410423160 after retry RPC after error@1312104668) -> (continuation of Deferred@1681817193 after retry RPC after error@1031141036), errback=(continuation of Deferred@925987235 after retry RPC after error@1506670098) -> (continuation of Deferred@1533478597 after retry RPC after error@2001463428) -> (continuation of Deferred@1271608231 after retry RPC after error@2105549226) -> (continuation of Deferred@506614257 after retry RPC after error@907380091) -> (continuation of Deferred@609340949 after retry RPC after error@1486890564) -> (continuation of Deferred@1410423160 after retry RPC after error@1312104668) -> (continuation of Deferred@1681817193 after retry RPC after error@1031141036)))
02:59:57.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:57.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:57.980 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.980 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.980 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.981 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:57.981 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.981 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:57.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.000 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.000 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.000 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.001 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.001 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.002 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.002 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.019 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.020 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.020 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.020 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.020 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.020 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.020 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.039 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.040 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.040 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.040 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.040 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.040 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.040 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.041 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.061 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.061 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.061 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.061 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.061 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.061 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.062 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.101 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.101 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.101 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.101 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.102 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.102 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.102 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.102 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.139 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.140 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.140 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.140 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.140 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.140 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.140 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.141 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.180 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.180 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.180 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.180 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.181 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.181 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.181 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1102229038(state=PENDING, result=null, callback=(continuation of Deferred@128079565 after retry RPC after error@1397639645) -> (continuation of Deferred@1886019799 after retry RPC after error@612197162) -> (continuation of Deferred@830229846 after retry RPC after error@274590213) -> (continuation of Deferred@865229825 after retry RPC after error@2084117866) -> (continuation of Deferred@805108038 after retry RPC after error@1343103870) -> (continuation of Deferred@1901924138 after retry RPC after error@1161535014) -> (continuation of Deferred@222656782 after retry RPC after error@745647627), errback=(continuation of Deferred@128079565 after retry RPC after error@1397639645) -> (continuation of Deferred@1886019799 after retry RPC after error@612197162) -> (continuation of Deferred@830229846 after retry RPC after error@274590213) -> (continuation of Deferred@865229825 after retry RPC after error@2084117866) -> (continuation of Deferred@805108038 after retry RPC after error@1343103870) -> (continuation of Deferred@1901924138 after retry RPC after error@1161535014) -> (continuation of Deferred@222656782 after retry RPC after error@745647627)))
02:59:58.182 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1102229038(state=PENDING, result=null, callback=(continuation of Deferred@128079565 after retry RPC after error@1397639645) -> (continuation of Deferred@1886019799 after retry RPC after error@612197162) -> (continuation of Deferred@830229846 after retry RPC after error@274590213) -> (continuation of Deferred@865229825 after retry RPC after error@2084117866) -> (continuation of Deferred@805108038 after retry RPC after error@1343103870) -> (continuation of Deferred@1901924138 after retry RPC after error@1161535014) -> (continuation of Deferred@222656782 after retry RPC after error@745647627), errback=(continuation of Deferred@128079565 after retry RPC after error@1397639645) -> (continuation of Deferred@1886019799 after retry RPC after error@612197162) -> (continuation of Deferred@830229846 after retry RPC after error@274590213) -> (continuation of Deferred@865229825 after retry RPC after error@2084117866) -> (continuation of Deferred@805108038 after retry RPC after error@1343103870) -> (continuation of Deferred@1901924138 after retry RPC after error@1161535014) -> (continuation of Deferred@222656782 after retry RPC after error@745647627)))
02:59:58.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.200 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.200 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.200 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.200 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.200 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.201 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.219 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.220 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.220 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.220 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.220 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.221 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.221 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.239 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.239 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.240 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.240 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.240 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.240 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.240 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.240 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.241 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.261 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.261 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.261 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.261 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.261 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.261 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.261 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.280 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.280 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.281 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.281 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.281 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.281 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.300 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.300 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.301 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.301 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.301 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.301 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.341 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.341 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.341 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.341 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.341 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.341 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.342 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.342 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2022464300(state=PENDING, result=null, callback=(continuation of Deferred@693430102 after retry RPC after error@38913842) -> (continuation of Deferred@258460083 after retry RPC after error@363374152) -> (continuation of Deferred@457739772 after retry RPC after error@397193712) -> (continuation of Deferred@649582823 after retry RPC after error@1996337311) -> (continuation of Deferred@696644817 after retry RPC after error@1532289824) -> (continuation of Deferred@1128032333 after retry RPC after error@1490960893), errback=(continuation of Deferred@693430102 after retry RPC after error@38913842) -> (continuation of Deferred@258460083 after retry RPC after error@363374152) -> (continuation of Deferred@457739772 after retry RPC after error@397193712) -> (continuation of Deferred@649582823 after retry RPC after error@1996337311) -> (continuation of Deferred@696644817 after retry RPC after error@1532289824) -> (continuation of Deferred@1128032333 after retry RPC after error@1490960893)))
02:59:58.342 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2022464300(state=PENDING, result=null, callback=(continuation of Deferred@693430102 after retry RPC after error@38913842) -> (continuation of Deferred@258460083 after retry RPC after error@363374152) -> (continuation of Deferred@457739772 after retry RPC after error@397193712) -> (continuation of Deferred@649582823 after retry RPC after error@1996337311) -> (continuation of Deferred@696644817 after retry RPC after error@1532289824) -> (continuation of Deferred@1128032333 after retry RPC after error@1490960893), errback=(continuation of Deferred@693430102 after retry RPC after error@38913842) -> (continuation of Deferred@258460083 after retry RPC after error@363374152) -> (continuation of Deferred@457739772 after retry RPC after error@397193712) -> (continuation of Deferred@649582823 after retry RPC after error@1996337311) -> (continuation of Deferred@696644817 after retry RPC after error@1532289824) -> (continuation of Deferred@1128032333 after retry RPC after error@1490960893)))
02:59:58.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.361 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.361 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.361 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.361 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.361 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.361 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.362 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.380 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.380 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.381 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.381 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.381 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.381 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.400 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.400 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.400 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.400 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.400 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.400 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.419 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.420 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.420 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.420 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.420 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.420 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.420 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.421 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.441 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.441 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.441 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.441 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.441 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.441 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.442 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.481 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.481 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.481 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.481 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.482 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.482 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.482 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.482 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.541 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.541 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.541 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.541 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.541 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.541 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.541 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.542 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.579 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.580 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.580 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.580 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.580 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.580 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.580 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.581 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2103828855(state=PENDING, result=null, callback=(continuation of Deferred@684529924 after retry RPC after error@1136964872) -> (continuation of Deferred@626233976 after retry RPC after error@2012197372) -> (continuation of Deferred@933921596 after retry RPC after error@1361058868) -> (continuation of Deferred@1759089722 after retry RPC after error@1655945356) -> (continuation of Deferred@23295109 after retry RPC after error@1349720353) -> (continuation of Deferred@1949249361 after retry RPC after error@187141760) -> (continuation of Deferred@293213245 after retry RPC after error@1493740834), errback=(continuation of Deferred@684529924 after retry RPC after error@1136964872) -> (continuation of Deferred@626233976 after retry RPC after error@2012197372) -> (continuation of Deferred@933921596 after retry RPC after error@1361058868) -> (continuation of Deferred@1759089722 after retry RPC after error@1655945356) -> (continuation of Deferred@23295109 after retry RPC after error@1349720353) -> (continuation of Deferred@1949249361 after retry RPC after error@187141760) -> (continuation of Deferred@293213245 after retry RPC after error@1493740834)))
02:59:58.582 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2103828855(state=PENDING, result=null, callback=(continuation of Deferred@684529924 after retry RPC after error@1136964872) -> (continuation of Deferred@626233976 after retry RPC after error@2012197372) -> (continuation of Deferred@933921596 after retry RPC after error@1361058868) -> (continuation of Deferred@1759089722 after retry RPC after error@1655945356) -> (continuation of Deferred@23295109 after retry RPC after error@1349720353) -> (continuation of Deferred@1949249361 after retry RPC after error@187141760) -> (continuation of Deferred@293213245 after retry RPC after error@1493740834), errback=(continuation of Deferred@684529924 after retry RPC after error@1136964872) -> (continuation of Deferred@626233976 after retry RPC after error@2012197372) -> (continuation of Deferred@933921596 after retry RPC after error@1361058868) -> (continuation of Deferred@1759089722 after retry RPC after error@1655945356) -> (continuation of Deferred@23295109 after retry RPC after error@1349720353) -> (continuation of Deferred@1949249361 after retry RPC after error@187141760) -> (continuation of Deferred@293213245 after retry RPC after error@1493740834)))
02:59:58.599 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.600 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.600 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.600 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.600 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.600 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.600 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.621 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.621 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.621 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.621 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.621 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.621 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.622 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.641 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.641 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.641 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.641 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.641 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.641 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.642 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.661 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.661 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.661 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.661 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.661 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.661 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.681 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.681 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.681 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.681 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.681 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.681 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.701 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.701 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.701 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.701 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.701 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.701 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.702 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.759 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.760 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.760 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.760 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.761 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.761 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.761 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.840 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.840 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.841 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.841 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.841 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.841 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.842 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.842 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1760722127(state=PENDING, result=null, callback=(continuation of Deferred@790250923 after retry RPC after error@2024124550) -> (continuation of Deferred@1565752350 after retry RPC after error@641496959) -> (continuation of Deferred@1542867214 after retry RPC after error@243458934) -> (continuation of Deferred@1215814964 after retry RPC after error@230022346) -> (continuation of Deferred@1692555851 after retry RPC after error@2103610610) -> (continuation of Deferred@185233431 after retry RPC after error@687517232) -> (continuation of Deferred@1581025638 after retry RPC after error@936418127), errback=(continuation of Deferred@790250923 after retry RPC after error@2024124550) -> (continuation of Deferred@1565752350 after retry RPC after error@641496959) -> (continuation of Deferred@1542867214 after retry RPC after error@243458934) -> (continuation of Deferred@1215814964 after retry RPC after error@230022346) -> (continuation of Deferred@1692555851 after retry RPC after error@2103610610) -> (continuation of Deferred@185233431 after retry RPC after error@687517232) -> (continuation of Deferred@1581025638 after retry RPC after error@936418127)))
02:59:58.842 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1760722127(state=PENDING, result=null, callback=(continuation of Deferred@790250923 after retry RPC after error@2024124550) -> (continuation of Deferred@1565752350 after retry RPC after error@641496959) -> (continuation of Deferred@1542867214 after retry RPC after error@243458934) -> (continuation of Deferred@1215814964 after retry RPC after error@230022346) -> (continuation of Deferred@1692555851 after retry RPC after error@2103610610) -> (continuation of Deferred@185233431 after retry RPC after error@687517232) -> (continuation of Deferred@1581025638 after retry RPC after error@936418127), errback=(continuation of Deferred@790250923 after retry RPC after error@2024124550) -> (continuation of Deferred@1565752350 after retry RPC after error@641496959) -> (continuation of Deferred@1542867214 after retry RPC after error@243458934) -> (continuation of Deferred@1215814964 after retry RPC after error@230022346) -> (continuation of Deferred@1692555851 after retry RPC after error@2103610610) -> (continuation of Deferred@185233431 after retry RPC after error@687517232) -> (continuation of Deferred@1581025638 after retry RPC after error@936418127)))
02:59:58.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.860 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.860 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.860 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.861 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.861 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.861 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.880 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.880 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.880 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.880 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.880 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.881 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.899 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.900 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.900 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.900 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.900 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.901 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.901 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.919 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.920 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.920 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.920 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.920 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.920 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.921 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.939 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.940 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.940 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.940 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.940 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.940 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.940 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:58.979 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:58.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:58.980 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.980 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.980 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:58.980 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.980 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.980 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:58.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.061 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.061 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.061 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.061 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.061 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.061 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.119 [DEBUG - pool-40-thread-1] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1304136013(state=PENDING, result=null, callback=(continuation of Deferred@956869572 after retry RPC after error@1276577846) -> (continuation of Deferred@439992101 after retry RPC after error@198814926) -> (continuation of Deferred@2012442874 after retry RPC after error@64065674) -> (continuation of Deferred@1422313282 after retry RPC after error@314556104) -> (continuation of Deferred@2116511941 after retry RPC after error@1912463947) -> (continuation of Deferred@333694945 after retry RPC after error@2084922027) -> (continuation of Deferred@971066534 after retry RPC after error@663752299), errback=(continuation of Deferred@956869572 after retry RPC after error@1276577846) -> (continuation of Deferred@439992101 after retry RPC after error@198814926) -> (continuation of Deferred@2012442874 after retry RPC after error@64065674) -> (continuation of Deferred@1422313282 after retry RPC after error@314556104) -> (continuation of Deferred@2116511941 after retry RPC after error@1912463947) -> (continuation of Deferred@333694945 after retry RPC after error@2084922027) -> (continuation of Deferred@971066534 after retry RPC after error@663752299)))
02:59:59.120 [WARN - pool-40-thread-1] (HashedWheelTimer.java:729) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
02:59:59.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.140 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.140 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.140 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.140 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.140 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.140 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.141 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.159 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.160 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.160 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.160 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.160 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.160 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.160 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.179 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.179 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.180 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.180 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.180 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.180 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.180 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.180 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.201 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.201 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.201 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.201 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.201 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.201 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.201 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.201 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.202 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.203 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.221 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.221 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.221 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.221 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.221 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.221 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.222 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.240 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.241 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.242 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.241 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.242 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.242 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.242 [WARN - kudu-nio-3] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.301 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.301 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.301 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.301 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.301 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.301 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.302 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.302 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] refreshing cache from master, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [64ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [163ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@914184529(state=PENDING, result=null, callback=(continuation of Deferred@794416535 after retry RPC after error@1211087789) -> (continuation of Deferred@1388688241 after retry RPC after error@246337449) -> (continuation of Deferred@1874482822 after retry RPC after error@1526557199) -> (continuation of Deferred@626787354 after retry RPC after error@1576486047) -> (continuation of Deferred@1623472065 after retry RPC after error@1373148356) -> (continuation of Deferred@350303793 after retry RPC after error@1934973838), errback=(continuation of Deferred@794416535 after retry RPC after error@1211087789) -> (continuation of Deferred@1388688241 after retry RPC after error@246337449) -> (continuation of Deferred@1874482822 after retry RPC after error@1526557199) -> (continuation of Deferred@626787354 after retry RPC after error@1576486047) -> (continuation of Deferred@1623472065 after retry RPC after error@1373148356) -> (continuation of Deferred@350303793 after retry RPC after error@1934973838)))
02:59:59.302 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] refreshing cache from master, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [64ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [163ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@914184529(state=PENDING, result=null, callback=(continuation of Deferred@794416535 after retry RPC after error@1211087789) -> (continuation of Deferred@1388688241 after retry RPC after error@246337449) -> (continuation of Deferred@1874482822 after retry RPC after error@1526557199) -> (continuation of Deferred@626787354 after retry RPC after error@1576486047) -> (continuation of Deferred@1623472065 after retry RPC after error@1373148356) -> (continuation of Deferred@350303793 after retry RPC after error@1934973838), errback=(continuation of Deferred@794416535 after retry RPC after error@1211087789) -> (continuation of Deferred@1388688241 after retry RPC after error@246337449) -> (continuation of Deferred@1874482822 after retry RPC after error@1526557199) -> (continuation of Deferred@626787354 after retry RPC after error@1576486047) -> (continuation of Deferred@1623472065 after retry RPC after error@1373148356) -> (continuation of Deferred@350303793 after retry RPC after error@1934973838)))
02:59:59.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.321 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.321 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.321 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.321 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.321 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.321 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.322 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.341 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.341 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.341 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.341 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.341 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.341 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.342 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.361 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.361 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.361 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.361 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.361 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.361 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.381 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.381 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.381 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.381 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.381 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.381 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.382 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.401 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.401 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.401 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.401 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.401 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.401 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.420 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.420 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.421 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.421 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.421 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.421 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.421 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.481 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.481 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.481 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.481 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.481 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.481 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.482 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.559 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.559 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.560 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.560 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.560 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.560 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.560 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.560 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.561 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.561 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@565457165(state=PENDING, result=null, callback=(continuation of Deferred@685319544 after retry RPC after error@1406713053) -> (continuation of Deferred@1962577969 after retry RPC after error@1950809082) -> (continuation of Deferred@357954287 after retry RPC after error@66951884) -> (continuation of Deferred@589752730 after retry RPC after error@1618763567) -> (continuation of Deferred@1510214469 after retry RPC after error@893440307) -> (continuation of Deferred@557745208 after retry RPC after error@599335447) -> (continuation of Deferred@1503484678 after retry RPC after error@1785188431), errback=(continuation of Deferred@685319544 after retry RPC after error@1406713053) -> (continuation of Deferred@1962577969 after retry RPC after error@1950809082) -> (continuation of Deferred@357954287 after retry RPC after error@66951884) -> (continuation of Deferred@589752730 after retry RPC after error@1618763567) -> (continuation of Deferred@1510214469 after retry RPC after error@893440307) -> (continuation of Deferred@557745208 after retry RPC after error@599335447) -> (continuation of Deferred@1503484678 after retry RPC after error@1785188431)))
02:59:59.561 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@565457165(state=PENDING, result=null, callback=(continuation of Deferred@685319544 after retry RPC after error@1406713053) -> (continuation of Deferred@1962577969 after retry RPC after error@1950809082) -> (continuation of Deferred@357954287 after retry RPC after error@66951884) -> (continuation of Deferred@589752730 after retry RPC after error@1618763567) -> (continuation of Deferred@1510214469 after retry RPC after error@893440307) -> (continuation of Deferred@557745208 after retry RPC after error@599335447) -> (continuation of Deferred@1503484678 after retry RPC after error@1785188431), errback=(continuation of Deferred@685319544 after retry RPC after error@1406713053) -> (continuation of Deferred@1962577969 after retry RPC after error@1950809082) -> (continuation of Deferred@357954287 after retry RPC after error@66951884) -> (continuation of Deferred@589752730 after retry RPC after error@1618763567) -> (continuation of Deferred@1510214469 after retry RPC after error@893440307) -> (continuation of Deferred@557745208 after retry RPC after error@599335447) -> (continuation of Deferred@1503484678 after retry RPC after error@1785188431)))
02:59:59.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.581 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.581 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.581 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.581 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.581 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.581 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.582 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.601 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.601 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.601 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.601 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.601 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.601 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.602 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.620 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.620 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.621 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.621 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.621 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.621 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.640 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.641 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.641 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.641 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.641 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.641 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.660 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.660 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.661 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.661 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.661 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.661 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.680 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.680 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.680 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.680 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.680 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.680 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.740 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.740 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.741 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.741 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.741 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.741 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.801 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.801 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.801 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.801 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.801 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.801 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.802 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.802 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1642340775(state=PENDING, result=null, callback=(continuation of Deferred@1094338822 after retry RPC after error@1664504264) -> (continuation of Deferred@534512658 after retry RPC after error@1601562455) -> (continuation of Deferred@1284831174 after retry RPC after error@503142285) -> (continuation of Deferred@1652436148 after retry RPC after error@306258163) -> (continuation of Deferred@2038489018 after retry RPC after error@246853176) -> (continuation of Deferred@499005654 after retry RPC after error@107201834) -> (continuation of Deferred@2036442963 after retry RPC after error@1109201193), errback=(continuation of Deferred@1094338822 after retry RPC after error@1664504264) -> (continuation of Deferred@534512658 after retry RPC after error@1601562455) -> (continuation of Deferred@1284831174 after retry RPC after error@503142285) -> (continuation of Deferred@1652436148 after retry RPC after error@306258163) -> (continuation of Deferred@2038489018 after retry RPC after error@246853176) -> (continuation of Deferred@499005654 after retry RPC after error@107201834) -> (continuation of Deferred@2036442963 after retry RPC after error@1109201193)))
02:59:59.802 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1642340775(state=PENDING, result=null, callback=(continuation of Deferred@1094338822 after retry RPC after error@1664504264) -> (continuation of Deferred@534512658 after retry RPC after error@1601562455) -> (continuation of Deferred@1284831174 after retry RPC after error@503142285) -> (continuation of Deferred@1652436148 after retry RPC after error@306258163) -> (continuation of Deferred@2038489018 after retry RPC after error@246853176) -> (continuation of Deferred@499005654 after retry RPC after error@107201834) -> (continuation of Deferred@2036442963 after retry RPC after error@1109201193), errback=(continuation of Deferred@1094338822 after retry RPC after error@1664504264) -> (continuation of Deferred@534512658 after retry RPC after error@1601562455) -> (continuation of Deferred@1284831174 after retry RPC after error@503142285) -> (continuation of Deferred@1652436148 after retry RPC after error@306258163) -> (continuation of Deferred@2038489018 after retry RPC after error@246853176) -> (continuation of Deferred@499005654 after retry RPC after error@107201834) -> (continuation of Deferred@2036442963 after retry RPC after error@1109201193)))
02:59:59.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.821 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.821 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.822 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.822 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.822 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.822 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.822 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.822 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.839 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.839 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.840 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.840 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.840 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.840 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.840 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.840 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.861 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.861 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.861 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.861 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.861 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.861 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.862 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.881 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.881 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.881 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.881 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.881 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.881 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.882 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.900 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.901 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.901 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.901 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.901 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.901 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.941 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.941 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.941 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.941 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.941 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.941 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
02:59:59.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
02:59:59.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
02:59:59.981 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.981 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.981 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.981 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.981 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
02:59:59.981 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
02:59:59.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.059 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.061 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.061 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.061 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.061 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.061 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.061 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.061 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.062 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.062 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@166400183(state=PENDING, result=null, callback=(continuation of Deferred@1715843875 after retry RPC after error@115273151) -> (continuation of Deferred@982420923 after retry RPC after error@1798393077) -> (continuation of Deferred@547759643 after retry RPC after error@1373934757) -> (continuation of Deferred@524888183 after retry RPC after error@2122226063) -> (continuation of Deferred@1870266021 after retry RPC after error@441261651) -> (continuation of Deferred@1276221437 after retry RPC after error@1177148689) -> (continuation of Deferred@1720029095 after retry RPC after error@348699298), errback=(continuation of Deferred@1715843875 after retry RPC after error@115273151) -> (continuation of Deferred@982420923 after retry RPC after error@1798393077) -> (continuation of Deferred@547759643 after retry RPC after error@1373934757) -> (continuation of Deferred@524888183 after retry RPC after error@2122226063) -> (continuation of Deferred@1870266021 after retry RPC after error@441261651) -> (continuation of Deferred@1276221437 after retry RPC after error@1177148689) -> (continuation of Deferred@1720029095 after retry RPC after error@348699298)))
03:00:00.063 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@166400183(state=PENDING, result=null, callback=(continuation of Deferred@1715843875 after retry RPC after error@115273151) -> (continuation of Deferred@982420923 after retry RPC after error@1798393077) -> (continuation of Deferred@547759643 after retry RPC after error@1373934757) -> (continuation of Deferred@524888183 after retry RPC after error@2122226063) -> (continuation of Deferred@1870266021 after retry RPC after error@441261651) -> (continuation of Deferred@1276221437 after retry RPC after error@1177148689) -> (continuation of Deferred@1720029095 after retry RPC after error@348699298), errback=(continuation of Deferred@1715843875 after retry RPC after error@115273151) -> (continuation of Deferred@982420923 after retry RPC after error@1798393077) -> (continuation of Deferred@547759643 after retry RPC after error@1373934757) -> (continuation of Deferred@524888183 after retry RPC after error@2122226063) -> (continuation of Deferred@1870266021 after retry RPC after error@441261651) -> (continuation of Deferred@1276221437 after retry RPC after error@1177148689) -> (continuation of Deferred@1720029095 after retry RPC after error@348699298)))
03:00:00.079 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.080 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.080 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.080 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.080 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.080 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.080 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.081 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.099 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.100 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.100 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.100 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.100 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.100 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.100 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.119 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.120 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.120 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.120 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.120 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.120 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.120 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.141 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.141 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.141 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.141 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.141 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.141 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.142 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.161 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.161 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.161 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.161 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.161 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.161 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.181 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.181 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.181 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.181 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.181 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.181 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.241 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.241 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.241 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.241 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.241 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.241 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.241 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.242 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.259 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.260 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.260 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.260 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.261 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.261 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.261 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.261 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.262 [DEBUG - kudu-nio-6] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [163ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1152429122(state=PENDING, result=null, callback=(continuation of Deferred@887457308 after retry RPC after error@320952817) -> (continuation of Deferred@1194599465 after retry RPC after error@111624753) -> (continuation of Deferred@510325151 after retry RPC after error@119393650) -> (continuation of Deferred@1643029716 after retry RPC after error@1566888978) -> (continuation of Deferred@348860456 after retry RPC after error@2111052190) -> (continuation of Deferred@1762188020 after retry RPC after error@314005379) -> (continuation of Deferred@1605398413 after retry RPC after error@41260300), errback=(continuation of Deferred@887457308 after retry RPC after error@320952817) -> (continuation of Deferred@1194599465 after retry RPC after error@111624753) -> (continuation of Deferred@510325151 after retry RPC after error@119393650) -> (continuation of Deferred@1643029716 after retry RPC after error@1566888978) -> (continuation of Deferred@348860456 after retry RPC after error@2111052190) -> (continuation of Deferred@1762188020 after retry RPC after error@314005379) -> (continuation of Deferred@1605398413 after retry RPC after error@41260300)))
03:00:00.262 [DEBUG - kudu-nio-6] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [163ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1152429122(state=PENDING, result=null, callback=(continuation of Deferred@887457308 after retry RPC after error@320952817) -> (continuation of Deferred@1194599465 after retry RPC after error@111624753) -> (continuation of Deferred@510325151 after retry RPC after error@119393650) -> (continuation of Deferred@1643029716 after retry RPC after error@1566888978) -> (continuation of Deferred@348860456 after retry RPC after error@2111052190) -> (continuation of Deferred@1762188020 after retry RPC after error@314005379) -> (continuation of Deferred@1605398413 after retry RPC after error@41260300), errback=(continuation of Deferred@887457308 after retry RPC after error@320952817) -> (continuation of Deferred@1194599465 after retry RPC after error@111624753) -> (continuation of Deferred@510325151 after retry RPC after error@119393650) -> (continuation of Deferred@1643029716 after retry RPC after error@1566888978) -> (continuation of Deferred@348860456 after retry RPC after error@2111052190) -> (continuation of Deferred@1762188020 after retry RPC after error@314005379) -> (continuation of Deferred@1605398413 after retry RPC after error@41260300)))
03:00:00.279 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.280 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.280 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.280 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.280 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.280 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.280 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.299 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.300 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.300 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.300 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.300 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.300 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.300 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.319 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.320 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.320 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.320 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.320 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.320 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.320 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.339 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.339 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.340 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.340 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.340 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.340 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.340 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.340 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.361 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.361 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.361 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.361 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.361 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.361 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.362 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.381 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.381 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.381 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.381 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.381 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.381 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.382 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.400 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.401 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.401 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.401 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.401 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.401 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.461 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.461 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.461 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.461 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.461 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.461 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.462 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.462 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2102225559(state=PENDING, result=null, callback=(continuation of Deferred@791259213 after retry RPC after error@1288291536) -> (continuation of Deferred@410869435 after retry RPC after error@250779377) -> (continuation of Deferred@1820011682 after retry RPC after error@1855359673) -> (continuation of Deferred@485263702 after retry RPC after error@112443684) -> (continuation of Deferred@857273196 after retry RPC after error@1761703156) -> (continuation of Deferred@291501755 after retry RPC after error@167620437) -> (continuation of Deferred@2106244757 after retry RPC after error@533648132), errback=(continuation of Deferred@791259213 after retry RPC after error@1288291536) -> (continuation of Deferred@410869435 after retry RPC after error@250779377) -> (continuation of Deferred@1820011682 after retry RPC after error@1855359673) -> (continuation of Deferred@485263702 after retry RPC after error@112443684) -> (continuation of Deferred@857273196 after retry RPC after error@1761703156) -> (continuation of Deferred@291501755 after retry RPC after error@167620437) -> (continuation of Deferred@2106244757 after retry RPC after error@533648132)))
03:00:00.462 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2102225559(state=PENDING, result=null, callback=(continuation of Deferred@791259213 after retry RPC after error@1288291536) -> (continuation of Deferred@410869435 after retry RPC after error@250779377) -> (continuation of Deferred@1820011682 after retry RPC after error@1855359673) -> (continuation of Deferred@485263702 after retry RPC after error@112443684) -> (continuation of Deferred@857273196 after retry RPC after error@1761703156) -> (continuation of Deferred@291501755 after retry RPC after error@167620437) -> (continuation of Deferred@2106244757 after retry RPC after error@533648132), errback=(continuation of Deferred@791259213 after retry RPC after error@1288291536) -> (continuation of Deferred@410869435 after retry RPC after error@250779377) -> (continuation of Deferred@1820011682 after retry RPC after error@1855359673) -> (continuation of Deferred@485263702 after retry RPC after error@112443684) -> (continuation of Deferred@857273196 after retry RPC after error@1761703156) -> (continuation of Deferred@291501755 after retry RPC after error@167620437) -> (continuation of Deferred@2106244757 after retry RPC after error@533648132)))
03:00:00.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.481 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.481 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.481 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.481 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.481 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.481 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.481 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.500 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.500 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.500 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.501 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.501 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.501 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.520 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.520 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.520 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.520 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.520 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.520 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.539 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.540 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.540 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.540 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.540 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.540 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.540 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.559 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.560 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.560 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.560 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.560 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.560 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.560 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.561 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.599 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.600 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.600 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.600 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.600 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.600 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.600 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.639 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.640 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.640 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.640 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.640 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.640 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.640 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.721 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.721 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.721 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.721 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.721 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.721 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.722 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.722 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1971877337(state=PENDING, result=null, callback=(continuation of Deferred@740823289 after retry RPC after error@1846813441) -> (continuation of Deferred@940689300 after retry RPC after error@234966002) -> (continuation of Deferred@1656892691 after retry RPC after error@755045985) -> (continuation of Deferred@454711602 after retry RPC after error@532733816) -> (continuation of Deferred@1822020390 after retry RPC after error@1225683658) -> (continuation of Deferred@1335568 after retry RPC after error@431379829) -> (continuation of Deferred@991212934 after retry RPC after error@28710924), errback=(continuation of Deferred@740823289 after retry RPC after error@1846813441) -> (continuation of Deferred@940689300 after retry RPC after error@234966002) -> (continuation of Deferred@1656892691 after retry RPC after error@755045985) -> (continuation of Deferred@454711602 after retry RPC after error@532733816) -> (continuation of Deferred@1822020390 after retry RPC after error@1225683658) -> (continuation of Deferred@1335568 after retry RPC after error@431379829) -> (continuation of Deferred@991212934 after retry RPC after error@28710924)))
03:00:00.723 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1971877337(state=PENDING, result=null, callback=(continuation of Deferred@740823289 after retry RPC after error@1846813441) -> (continuation of Deferred@940689300 after retry RPC after error@234966002) -> (continuation of Deferred@1656892691 after retry RPC after error@755045985) -> (continuation of Deferred@454711602 after retry RPC after error@532733816) -> (continuation of Deferred@1822020390 after retry RPC after error@1225683658) -> (continuation of Deferred@1335568 after retry RPC after error@431379829) -> (continuation of Deferred@991212934 after retry RPC after error@28710924), errback=(continuation of Deferred@740823289 after retry RPC after error@1846813441) -> (continuation of Deferred@940689300 after retry RPC after error@234966002) -> (continuation of Deferred@1656892691 after retry RPC after error@755045985) -> (continuation of Deferred@454711602 after retry RPC after error@532733816) -> (continuation of Deferred@1822020390 after retry RPC after error@1225683658) -> (continuation of Deferred@1335568 after retry RPC after error@431379829) -> (continuation of Deferred@991212934 after retry RPC after error@28710924)))
03:00:00.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.740 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.740 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.741 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.741 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.741 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.741 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.760 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.760 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.760 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.760 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.760 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.760 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.779 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.780 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.780 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.780 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.780 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.780 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.780 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.799 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.800 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.800 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.800 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.800 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.800 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.800 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.819 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.820 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.820 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.820 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.821 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.821 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.821 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.859 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.860 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.860 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.860 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.860 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.860 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.860 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.940 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.940 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.941 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.941 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.941 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.941 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.942 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1259650828(state=PENDING, result=null, callback=(continuation of Deferred@986841325 after retry RPC after error@1685912076) -> (continuation of Deferred@871894155 after retry RPC after error@1568291111) -> (continuation of Deferred@1996853005 after retry RPC after error@1957341051) -> (continuation of Deferred@130380081 after retry RPC after error@200983535) -> (continuation of Deferred@1348225146 after retry RPC after error@1902947445) -> (continuation of Deferred@1561069456 after retry RPC after error@11813477), errback=(continuation of Deferred@986841325 after retry RPC after error@1685912076) -> (continuation of Deferred@871894155 after retry RPC after error@1568291111) -> (continuation of Deferred@1996853005 after retry RPC after error@1957341051) -> (continuation of Deferred@130380081 after retry RPC after error@200983535) -> (continuation of Deferred@1348225146 after retry RPC after error@1902947445) -> (continuation of Deferred@1561069456 after retry RPC after error@11813477)))
03:00:00.942 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1259650828(state=PENDING, result=null, callback=(continuation of Deferred@986841325 after retry RPC after error@1685912076) -> (continuation of Deferred@871894155 after retry RPC after error@1568291111) -> (continuation of Deferred@1996853005 after retry RPC after error@1957341051) -> (continuation of Deferred@130380081 after retry RPC after error@200983535) -> (continuation of Deferred@1348225146 after retry RPC after error@1902947445) -> (continuation of Deferred@1561069456 after retry RPC after error@11813477), errback=(continuation of Deferred@986841325 after retry RPC after error@1685912076) -> (continuation of Deferred@871894155 after retry RPC after error@1568291111) -> (continuation of Deferred@1996853005 after retry RPC after error@1957341051) -> (continuation of Deferred@130380081 after retry RPC after error@200983535) -> (continuation of Deferred@1348225146 after retry RPC after error@1902947445) -> (continuation of Deferred@1561069456 after retry RPC after error@11813477)))
03:00:00.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.960 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.960 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.960 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.960 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.960 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.960 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.979 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:00.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:00.980 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.980 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.980 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:00.981 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.981 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.981 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:00.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:00.999 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.000 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.000 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.000 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.000 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.001 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.001 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.021 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.021 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.021 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.021 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.021 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.021 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.022 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.039 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.040 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.040 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.040 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.040 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.040 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.040 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.041 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.061 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.061 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.061 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.061 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.061 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.061 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.061 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.081 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.081 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.081 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.081 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.081 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.081 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.082 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.220 [DEBUG - pool-40-thread-1] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@23184525(state=PENDING, result=null, callback=(continuation of Deferred@768649229 after retry RPC after error@1480038021) -> (continuation of Deferred@251622374 after retry RPC after error@20286232) -> (continuation of Deferred@688774131 after retry RPC after error@1298854889) -> (continuation of Deferred@947902945 after retry RPC after error@364902589) -> (continuation of Deferred@1386904506 after retry RPC after error@1212497853) -> (continuation of Deferred@238245765 after retry RPC after error@1680026862) -> (continuation of Deferred@1700129915 after retry RPC after error@76324795), errback=(continuation of Deferred@768649229 after retry RPC after error@1480038021) -> (continuation of Deferred@251622374 after retry RPC after error@20286232) -> (continuation of Deferred@688774131 after retry RPC after error@1298854889) -> (continuation of Deferred@947902945 after retry RPC after error@364902589) -> (continuation of Deferred@1386904506 after retry RPC after error@1212497853) -> (continuation of Deferred@238245765 after retry RPC after error@1680026862) -> (continuation of Deferred@1700129915 after retry RPC after error@76324795)))
03:00:01.220 [WARN - pool-40-thread-1] (HashedWheelTimer.java:729) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
03:00:01.239 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.240 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.240 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.240 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.240 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.240 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.240 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.241 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.261 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.261 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.261 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.261 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.261 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.261 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.262 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.281 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.281 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.281 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.281 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.281 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.281 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.281 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.282 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.301 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.301 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.301 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.301 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.301 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.301 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.302 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.321 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.321 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.321 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.321 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.321 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.321 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.340 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.340 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.340 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.340 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.340 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.340 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.401 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.401 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.401 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.401 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.401 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.401 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.481 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.481 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.481 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.481 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.481 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.481 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.481 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.482 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.482 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@637892806(state=PENDING, result=null, callback=(continuation of Deferred@1976292495 after retry RPC after error@1652134767) -> (continuation of Deferred@1874169627 after retry RPC after error@1660619107) -> (continuation of Deferred@1237308959 after retry RPC after error@1744029356) -> (continuation of Deferred@620580894 after retry RPC after error@511174131) -> (continuation of Deferred@693098252 after retry RPC after error@1349558831) -> (continuation of Deferred@507737503 after retry RPC after error@1060787161) -> (continuation of Deferred@1746539653 after retry RPC after error@1034859123), errback=(continuation of Deferred@1976292495 after retry RPC after error@1652134767) -> (continuation of Deferred@1874169627 after retry RPC after error@1660619107) -> (continuation of Deferred@1237308959 after retry RPC after error@1744029356) -> (continuation of Deferred@620580894 after retry RPC after error@511174131) -> (continuation of Deferred@693098252 after retry RPC after error@1349558831) -> (continuation of Deferred@507737503 after retry RPC after error@1060787161) -> (continuation of Deferred@1746539653 after retry RPC after error@1034859123)))
03:00:01.483 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@637892806(state=PENDING, result=null, callback=(continuation of Deferred@1976292495 after retry RPC after error@1652134767) -> (continuation of Deferred@1874169627 after retry RPC after error@1660619107) -> (continuation of Deferred@1237308959 after retry RPC after error@1744029356) -> (continuation of Deferred@620580894 after retry RPC after error@511174131) -> (continuation of Deferred@693098252 after retry RPC after error@1349558831) -> (continuation of Deferred@507737503 after retry RPC after error@1060787161) -> (continuation of Deferred@1746539653 after retry RPC after error@1034859123), errback=(continuation of Deferred@1976292495 after retry RPC after error@1652134767) -> (continuation of Deferred@1874169627 after retry RPC after error@1660619107) -> (continuation of Deferred@1237308959 after retry RPC after error@1744029356) -> (continuation of Deferred@620580894 after retry RPC after error@511174131) -> (continuation of Deferred@693098252 after retry RPC after error@1349558831) -> (continuation of Deferred@507737503 after retry RPC after error@1060787161) -> (continuation of Deferred@1746539653 after retry RPC after error@1034859123)))
03:00:01.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.500 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.501 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.501 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.501 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.501 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.501 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.520 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.520 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.520 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.520 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.521 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.521 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.539 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.540 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.540 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.540 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.540 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.540 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.540 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.559 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.560 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.560 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.560 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.560 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.560 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.560 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.561 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.579 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.580 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.580 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.580 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.580 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.580 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.580 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.619 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.620 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.620 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.620 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.620 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.620 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.620 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.659 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.660 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.660 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.660 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.660 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.660 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.660 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.720 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.720 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.720 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.721 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.721 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.721 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.722 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.722 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1628693474(state=PENDING, result=null, callback=(continuation of Deferred@222737094 after retry RPC after error@16155619) -> (continuation of Deferred@821890339 after retry RPC after error@1318057005) -> (continuation of Deferred@1994895288 after retry RPC after error@1119535900) -> (continuation of Deferred@452348541 after retry RPC after error@736750482) -> (continuation of Deferred@1622674232 after retry RPC after error@1955497488) -> (continuation of Deferred@1543741278 after retry RPC after error@944917047) -> (continuation of Deferred@403924942 after retry RPC after error@405654746), errback=(continuation of Deferred@222737094 after retry RPC after error@16155619) -> (continuation of Deferred@821890339 after retry RPC after error@1318057005) -> (continuation of Deferred@1994895288 after retry RPC after error@1119535900) -> (continuation of Deferred@452348541 after retry RPC after error@736750482) -> (continuation of Deferred@1622674232 after retry RPC after error@1955497488) -> (continuation of Deferred@1543741278 after retry RPC after error@944917047) -> (continuation of Deferred@403924942 after retry RPC after error@405654746)))
03:00:01.723 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1628693474(state=PENDING, result=null, callback=(continuation of Deferred@222737094 after retry RPC after error@16155619) -> (continuation of Deferred@821890339 after retry RPC after error@1318057005) -> (continuation of Deferred@1994895288 after retry RPC after error@1119535900) -> (continuation of Deferred@452348541 after retry RPC after error@736750482) -> (continuation of Deferred@1622674232 after retry RPC after error@1955497488) -> (continuation of Deferred@1543741278 after retry RPC after error@944917047) -> (continuation of Deferred@403924942 after retry RPC after error@405654746), errback=(continuation of Deferred@222737094 after retry RPC after error@16155619) -> (continuation of Deferred@821890339 after retry RPC after error@1318057005) -> (continuation of Deferred@1994895288 after retry RPC after error@1119535900) -> (continuation of Deferred@452348541 after retry RPC after error@736750482) -> (continuation of Deferred@1622674232 after retry RPC after error@1955497488) -> (continuation of Deferred@1543741278 after retry RPC after error@944917047) -> (continuation of Deferred@403924942 after retry RPC after error@405654746)))
03:00:01.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.740 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.740 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.740 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.740 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.740 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.740 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.760 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.760 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.760 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.760 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.761 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.761 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.779 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.779 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.780 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.780 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.780 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.780 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.780 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.780 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.801 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.801 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.801 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.801 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.801 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.801 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.802 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.820 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.821 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.821 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.821 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.821 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.821 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.840 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.840 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.840 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.841 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.841 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.841 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.860 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.860 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.861 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.861 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.861 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.861 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.920 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.921 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.921 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.921 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.921 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.921 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.922 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.922 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2098146401(state=PENDING, result=null, callback=(continuation of Deferred@1656028178 after retry RPC after error@267907420) -> (continuation of Deferred@2128276151 after retry RPC after error@2058530089) -> (continuation of Deferred@1067686401 after retry RPC after error@258929080) -> (continuation of Deferred@642268592 after retry RPC after error@1232865205) -> (continuation of Deferred@1781333769 after retry RPC after error@305606853) -> (continuation of Deferred@1979001486 after retry RPC after error@1553844778), errback=(continuation of Deferred@1656028178 after retry RPC after error@267907420) -> (continuation of Deferred@2128276151 after retry RPC after error@2058530089) -> (continuation of Deferred@1067686401 after retry RPC after error@258929080) -> (continuation of Deferred@642268592 after retry RPC after error@1232865205) -> (continuation of Deferred@1781333769 after retry RPC after error@305606853) -> (continuation of Deferred@1979001486 after retry RPC after error@1553844778)))
03:00:01.922 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2098146401(state=PENDING, result=null, callback=(continuation of Deferred@1656028178 after retry RPC after error@267907420) -> (continuation of Deferred@2128276151 after retry RPC after error@2058530089) -> (continuation of Deferred@1067686401 after retry RPC after error@258929080) -> (continuation of Deferred@642268592 after retry RPC after error@1232865205) -> (continuation of Deferred@1781333769 after retry RPC after error@305606853) -> (continuation of Deferred@1979001486 after retry RPC after error@1553844778), errback=(continuation of Deferred@1656028178 after retry RPC after error@267907420) -> (continuation of Deferred@2128276151 after retry RPC after error@2058530089) -> (continuation of Deferred@1067686401 after retry RPC after error@258929080) -> (continuation of Deferred@642268592 after retry RPC after error@1232865205) -> (continuation of Deferred@1781333769 after retry RPC after error@305606853) -> (continuation of Deferred@1979001486 after retry RPC after error@1553844778)))
03:00:01.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.940 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.940 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.941 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.941 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.941 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.941 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.960 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.960 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.960 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.960 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.960 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.960 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.979 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:01.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:01.980 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.980 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.980 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.980 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:01.980 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.980 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:01.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:01.999 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.000 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.000 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.000 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.000 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.000 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.000 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.019 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.020 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.020 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.020 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.020 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.020 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.020 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.059 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.060 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.060 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.060 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.060 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.060 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.060 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.081 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.081 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.081 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.081 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.081 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.081 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.082 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.121 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.121 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.121 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.121 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.121 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.121 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.122 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [0ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1627253519(state=PENDING, result=null, callback=(continuation of Deferred@1780918259 after retry RPC after error@1394061143) -> (continuation of Deferred@705413075 after retry RPC after error@631344142) -> (continuation of Deferred@2062003830 after retry RPC after error@350542698) -> (continuation of Deferred@1667631874 after retry RPC after error@1250835100) -> (continuation of Deferred@224246181 after retry RPC after error@1062210799) -> (continuation of Deferred@1815948828 after retry RPC after error@1465689053) -> (continuation of Deferred@925673474 after retry RPC after error@145030524), errback=(continuation of Deferred@1780918259 after retry RPC after error@1394061143) -> (continuation of Deferred@705413075 after retry RPC after error@631344142) -> (continuation of Deferred@2062003830 after retry RPC after error@350542698) -> (continuation of Deferred@1667631874 after retry RPC after error@1250835100) -> (continuation of Deferred@224246181 after retry RPC after error@1062210799) -> (continuation of Deferred@1815948828 after retry RPC after error@1465689053) -> (continuation of Deferred@925673474 after retry RPC after error@145030524)))
03:00:02.122 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [0ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1627253519(state=PENDING, result=null, callback=(continuation of Deferred@1780918259 after retry RPC after error@1394061143) -> (continuation of Deferred@705413075 after retry RPC after error@631344142) -> (continuation of Deferred@2062003830 after retry RPC after error@350542698) -> (continuation of Deferred@1667631874 after retry RPC after error@1250835100) -> (continuation of Deferred@224246181 after retry RPC after error@1062210799) -> (continuation of Deferred@1815948828 after retry RPC after error@1465689053) -> (continuation of Deferred@925673474 after retry RPC after error@145030524), errback=(continuation of Deferred@1780918259 after retry RPC after error@1394061143) -> (continuation of Deferred@705413075 after retry RPC after error@631344142) -> (continuation of Deferred@2062003830 after retry RPC after error@350542698) -> (continuation of Deferred@1667631874 after retry RPC after error@1250835100) -> (continuation of Deferred@224246181 after retry RPC after error@1062210799) -> (continuation of Deferred@1815948828 after retry RPC after error@1465689053) -> (continuation of Deferred@925673474 after retry RPC after error@145030524)))
03:00:02.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.141 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.141 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.141 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.141 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.141 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.141 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.141 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.160 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.160 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.160 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.161 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.161 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.161 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.180 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.180 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.181 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.181 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.181 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.181 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.200 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.200 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.200 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.200 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.200 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.200 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.219 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.220 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.220 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.220 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.220 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.220 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.220 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.241 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.241 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.241 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.241 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.241 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.241 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.242 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.261 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.261 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.261 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.261 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.261 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.261 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.262 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.281 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.281 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.281 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.281 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.281 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.281 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.282 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.380 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.380 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.380 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.381 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.381 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.381 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.382 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.382 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@742251856(state=PENDING, result=null, callback=(continuation of Deferred@1883313977 after retry RPC after error@138246413) -> (continuation of Deferred@1567781268 after retry RPC after error@526901303) -> (continuation of Deferred@1734853693 after retry RPC after error@74529997) -> (continuation of Deferred@1254167302 after retry RPC after error@1787241521) -> (continuation of Deferred@857228144 after retry RPC after error@1040357002) -> (continuation of Deferred@2082073455 after retry RPC after error@804056689) -> (continuation of Deferred@1587424063 after retry RPC after error@1946308768) -> (continuation of Deferred@1207144412 after retry RPC after error@254045749), errback=(continuation of Deferred@1883313977 after retry RPC after error@138246413) -> (continuation of Deferred@1567781268 after retry RPC after error@526901303) -> (continuation of Deferred@1734853693 after retry RPC after error@74529997) -> (continuation of Deferred@1254167302 after retry RPC after error@1787241521) -> (continuation of Deferred@857228144 after retry RPC after error@1040357002) -> (continuation of Deferred@2082073455 after retry RPC after error@804056689) -> (continuation of Deferred@1587424063 after retry RPC after error@1946308768) -> (continuation of Deferred@1207144412 after retry RPC after error@254045749)))
03:00:02.382 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@742251856(state=PENDING, result=null, callback=(continuation of Deferred@1883313977 after retry RPC after error@138246413) -> (continuation of Deferred@1567781268 after retry RPC after error@526901303) -> (continuation of Deferred@1734853693 after retry RPC after error@74529997) -> (continuation of Deferred@1254167302 after retry RPC after error@1787241521) -> (continuation of Deferred@857228144 after retry RPC after error@1040357002) -> (continuation of Deferred@2082073455 after retry RPC after error@804056689) -> (continuation of Deferred@1587424063 after retry RPC after error@1946308768) -> (continuation of Deferred@1207144412 after retry RPC after error@254045749), errback=(continuation of Deferred@1883313977 after retry RPC after error@138246413) -> (continuation of Deferred@1567781268 after retry RPC after error@526901303) -> (continuation of Deferred@1734853693 after retry RPC after error@74529997) -> (continuation of Deferred@1254167302 after retry RPC after error@1787241521) -> (continuation of Deferred@857228144 after retry RPC after error@1040357002) -> (continuation of Deferred@2082073455 after retry RPC after error@804056689) -> (continuation of Deferred@1587424063 after retry RPC after error@1946308768) -> (continuation of Deferred@1207144412 after retry RPC after error@254045749)))
03:00:02.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.400 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.400 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.400 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.400 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.400 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.401 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.419 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.420 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.420 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.420 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.420 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.420 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.421 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.421 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.439 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.440 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.440 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.440 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.440 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.440 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.440 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.459 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.460 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.460 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.460 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.460 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.460 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.460 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.461 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.481 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.481 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.481 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.481 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.481 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.481 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.482 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.520 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.521 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.521 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.521 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.521 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.521 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.540 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.540 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.540 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.540 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.541 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.541 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.541 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1926513264(state=PENDING, result=null, callback=(continuation of Deferred@1502916470 after retry RPC after error@515709092) -> (continuation of Deferred@965721279 after retry RPC after error@1763225148) -> (continuation of Deferred@361448310 after retry RPC after error@1661276507) -> (continuation of Deferred@665531328 after retry RPC after error@200147015) -> (continuation of Deferred@1152479935 after retry RPC after error@259341499) -> (continuation of Deferred@1908292518 after retry RPC after error@178909678), errback=(continuation of Deferred@1502916470 after retry RPC after error@515709092) -> (continuation of Deferred@965721279 after retry RPC after error@1763225148) -> (continuation of Deferred@361448310 after retry RPC after error@1661276507) -> (continuation of Deferred@665531328 after retry RPC after error@200147015) -> (continuation of Deferred@1152479935 after retry RPC after error@259341499) -> (continuation of Deferred@1908292518 after retry RPC after error@178909678)))
03:00:02.542 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1926513264(state=PENDING, result=null, callback=(continuation of Deferred@1502916470 after retry RPC after error@515709092) -> (continuation of Deferred@965721279 after retry RPC after error@1763225148) -> (continuation of Deferred@361448310 after retry RPC after error@1661276507) -> (continuation of Deferred@665531328 after retry RPC after error@200147015) -> (continuation of Deferred@1152479935 after retry RPC after error@259341499) -> (continuation of Deferred@1908292518 after retry RPC after error@178909678), errback=(continuation of Deferred@1502916470 after retry RPC after error@515709092) -> (continuation of Deferred@965721279 after retry RPC after error@1763225148) -> (continuation of Deferred@361448310 after retry RPC after error@1661276507) -> (continuation of Deferred@665531328 after retry RPC after error@200147015) -> (continuation of Deferred@1152479935 after retry RPC after error@259341499) -> (continuation of Deferred@1908292518 after retry RPC after error@178909678)))
03:00:02.559 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.560 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.560 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.560 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.560 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.560 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.560 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.561 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.579 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.580 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.580 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.580 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.580 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.580 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.580 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.601 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.601 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.601 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.601 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.601 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.601 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.602 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.621 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.621 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.621 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.621 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.621 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.621 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.641 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.641 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.641 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.641 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.641 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.641 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.681 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.681 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.681 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.682 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.682 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.682 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.682 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.721 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.721 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.721 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.721 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.721 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.721 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.722 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.741 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.741 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.741 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.741 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.741 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.741 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.801 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.801 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.801 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.801 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.801 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.801 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.801 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.802 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.802 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [163ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@858808952(state=PENDING, result=null, callback=(continuation of Deferred@1783316514 after retry RPC after error@1800634777) -> (continuation of Deferred@823279235 after retry RPC after error@25278843) -> (continuation of Deferred@830190393 after retry RPC after error@1188275142) -> (continuation of Deferred@1721072569 after retry RPC after error@1783848049) -> (continuation of Deferred@106813561 after retry RPC after error@1573619431) -> (continuation of Deferred@273577556 after retry RPC after error@1619919096) -> (continuation of Deferred@150508160 after retry RPC after error@151376326), errback=(continuation of Deferred@1783316514 after retry RPC after error@1800634777) -> (continuation of Deferred@823279235 after retry RPC after error@25278843) -> (continuation of Deferred@830190393 after retry RPC after error@1188275142) -> (continuation of Deferred@1721072569 after retry RPC after error@1783848049) -> (continuation of Deferred@106813561 after retry RPC after error@1573619431) -> (continuation of Deferred@273577556 after retry RPC after error@1619919096) -> (continuation of Deferred@150508160 after retry RPC after error@151376326)))
03:00:02.802 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [163ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@858808952(state=PENDING, result=null, callback=(continuation of Deferred@1783316514 after retry RPC after error@1800634777) -> (continuation of Deferred@823279235 after retry RPC after error@25278843) -> (continuation of Deferred@830190393 after retry RPC after error@1188275142) -> (continuation of Deferred@1721072569 after retry RPC after error@1783848049) -> (continuation of Deferred@106813561 after retry RPC after error@1573619431) -> (continuation of Deferred@273577556 after retry RPC after error@1619919096) -> (continuation of Deferred@150508160 after retry RPC after error@151376326), errback=(continuation of Deferred@1783316514 after retry RPC after error@1800634777) -> (continuation of Deferred@823279235 after retry RPC after error@25278843) -> (continuation of Deferred@830190393 after retry RPC after error@1188275142) -> (continuation of Deferred@1721072569 after retry RPC after error@1783848049) -> (continuation of Deferred@106813561 after retry RPC after error@1573619431) -> (continuation of Deferred@273577556 after retry RPC after error@1619919096) -> (continuation of Deferred@150508160 after retry RPC after error@151376326)))
03:00:02.819 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.820 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.820 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.820 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.820 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.820 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.820 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.841 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.841 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.841 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.841 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.841 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.841 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.842 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.861 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.861 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.861 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.861 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.861 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.861 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.862 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.881 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.881 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.881 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.882 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.882 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.882 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.883 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.900 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.900 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.901 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.901 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.901 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.901 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.939 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:02.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:02.940 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.940 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.940 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:02.940 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.941 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.941 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:02.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:02.999 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.007 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.007 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.008 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.008 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.008 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.008 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.008 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.008 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.008 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=188), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [64ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [188ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [189ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [189ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [189ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@405073650(state=PENDING, result=null, callback=(continuation of Deferred@1676416864 after retry RPC after error@1198944088) -> (continuation of Deferred@987334978 after retry RPC after error@87401365) -> (continuation of Deferred@115471172 after retry RPC after error@1922650223) -> (continuation of Deferred@83645671 after retry RPC after error@754804454) -> (continuation of Deferred@630184942 after retry RPC after error@489553176) -> (continuation of Deferred@2064697654 after retry RPC after error@1016945380), errback=(continuation of Deferred@1676416864 after retry RPC after error@1198944088) -> (continuation of Deferred@987334978 after retry RPC after error@87401365) -> (continuation of Deferred@115471172 after retry RPC after error@1922650223) -> (continuation of Deferred@83645671 after retry RPC after error@754804454) -> (continuation of Deferred@630184942 after retry RPC after error@489553176) -> (continuation of Deferred@2064697654 after retry RPC after error@1016945380)))
03:00:03.009 [DEBUG - kudu-nio-2] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=188), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [64ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [188ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [189ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [189ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [189ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@405073650(state=PENDING, result=null, callback=(continuation of Deferred@1676416864 after retry RPC after error@1198944088) -> (continuation of Deferred@987334978 after retry RPC after error@87401365) -> (continuation of Deferred@115471172 after retry RPC after error@1922650223) -> (continuation of Deferred@83645671 after retry RPC after error@754804454) -> (continuation of Deferred@630184942 after retry RPC after error@489553176) -> (continuation of Deferred@2064697654 after retry RPC after error@1016945380), errback=(continuation of Deferred@1676416864 after retry RPC after error@1198944088) -> (continuation of Deferred@987334978 after retry RPC after error@87401365) -> (continuation of Deferred@115471172 after retry RPC after error@1922650223) -> (continuation of Deferred@83645671 after retry RPC after error@754804454) -> (continuation of Deferred@630184942 after retry RPC after error@489553176) -> (continuation of Deferred@2064697654 after retry RPC after error@1016945380)))
03:00:03.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.020 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.020 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.021 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.021 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.021 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.021 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.041 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.041 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.041 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.041 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.041 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.041 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.042 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.061 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.061 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.061 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.061 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.061 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.062 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.062 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.062 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.080 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.080 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.080 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.081 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.081 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.081 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.081 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.099 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.100 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.100 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.100 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.100 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.101 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.101 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.121 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.121 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.121 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.122 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.122 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.122 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.122 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.161 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.161 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.161 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.161 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.161 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.161 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.161 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.162 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.261 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.261 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.261 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.261 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.261 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.261 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.262 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.262 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [0ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@2100653496(state=PENDING, result=null, callback=(continuation of Deferred@1998358840 after retry RPC after error@1991116131) -> (continuation of Deferred@441612097 after retry RPC after error@1281134878) -> (continuation of Deferred@1603966180 after retry RPC after error@1494812521) -> (continuation of Deferred@1358470004 after retry RPC after error@2013537300) -> (continuation of Deferred@1941086788 after retry RPC after error@1644286228) -> (continuation of Deferred@1991689499 after retry RPC after error@801864541) -> (continuation of Deferred@1225608290 after retry RPC after error@636005670), errback=(continuation of Deferred@1998358840 after retry RPC after error@1991116131) -> (continuation of Deferred@441612097 after retry RPC after error@1281134878) -> (continuation of Deferred@1603966180 after retry RPC after error@1494812521) -> (continuation of Deferred@1358470004 after retry RPC after error@2013537300) -> (continuation of Deferred@1941086788 after retry RPC after error@1644286228) -> (continuation of Deferred@1991689499 after retry RPC after error@801864541) -> (continuation of Deferred@1225608290 after retry RPC after error@636005670)))
03:00:03.263 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [0ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@2100653496(state=PENDING, result=null, callback=(continuation of Deferred@1998358840 after retry RPC after error@1991116131) -> (continuation of Deferred@441612097 after retry RPC after error@1281134878) -> (continuation of Deferred@1603966180 after retry RPC after error@1494812521) -> (continuation of Deferred@1358470004 after retry RPC after error@2013537300) -> (continuation of Deferred@1941086788 after retry RPC after error@1644286228) -> (continuation of Deferred@1991689499 after retry RPC after error@801864541) -> (continuation of Deferred@1225608290 after retry RPC after error@636005670), errback=(continuation of Deferred@1998358840 after retry RPC after error@1991116131) -> (continuation of Deferred@441612097 after retry RPC after error@1281134878) -> (continuation of Deferred@1603966180 after retry RPC after error@1494812521) -> (continuation of Deferred@1358470004 after retry RPC after error@2013537300) -> (continuation of Deferred@1941086788 after retry RPC after error@1644286228) -> (continuation of Deferred@1991689499 after retry RPC after error@801864541) -> (continuation of Deferred@1225608290 after retry RPC after error@636005670)))
03:00:03.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.281 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.281 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.281 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.281 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.281 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.281 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.282 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.299 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.300 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.300 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.300 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.300 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.300 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.300 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.321 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.321 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.321 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.321 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.321 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.321 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.322 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.341 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.341 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.341 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.341 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.341 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.341 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.342 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.361 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.361 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.361 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.361 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.361 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.361 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.362 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.381 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.381 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.381 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.381 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.381 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.381 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.400 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.400 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.401 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.401 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.401 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.401 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.481 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.481 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.481 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.481 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.481 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.481 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.482 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.482 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1651484868(state=PENDING, result=null, callback=(continuation of Deferred@701342060 after retry RPC after error@698212456) -> (continuation of Deferred@304912305 after retry RPC after error@219817144) -> (continuation of Deferred@1234711222 after retry RPC after error@264246605) -> (continuation of Deferred@2041027099 after retry RPC after error@633626227) -> (continuation of Deferred@767673855 after retry RPC after error@581224311) -> (continuation of Deferred@1729554042 after retry RPC after error@1621498669) -> (continuation of Deferred@1139528955 after retry RPC after error@1206926478), errback=(continuation of Deferred@701342060 after retry RPC after error@698212456) -> (continuation of Deferred@304912305 after retry RPC after error@219817144) -> (continuation of Deferred@1234711222 after retry RPC after error@264246605) -> (continuation of Deferred@2041027099 after retry RPC after error@633626227) -> (continuation of Deferred@767673855 after retry RPC after error@581224311) -> (continuation of Deferred@1729554042 after retry RPC after error@1621498669) -> (continuation of Deferred@1139528955 after retry RPC after error@1206926478)))
03:00:03.483 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1651484868(state=PENDING, result=null, callback=(continuation of Deferred@701342060 after retry RPC after error@698212456) -> (continuation of Deferred@304912305 after retry RPC after error@219817144) -> (continuation of Deferred@1234711222 after retry RPC after error@264246605) -> (continuation of Deferred@2041027099 after retry RPC after error@633626227) -> (continuation of Deferred@767673855 after retry RPC after error@581224311) -> (continuation of Deferred@1729554042 after retry RPC after error@1621498669) -> (continuation of Deferred@1139528955 after retry RPC after error@1206926478), errback=(continuation of Deferred@701342060 after retry RPC after error@698212456) -> (continuation of Deferred@304912305 after retry RPC after error@219817144) -> (continuation of Deferred@1234711222 after retry RPC after error@264246605) -> (continuation of Deferred@2041027099 after retry RPC after error@633626227) -> (continuation of Deferred@767673855 after retry RPC after error@581224311) -> (continuation of Deferred@1729554042 after retry RPC after error@1621498669) -> (continuation of Deferred@1139528955 after retry RPC after error@1206926478)))
03:00:03.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.501 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.501 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.501 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.501 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.501 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.501 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.502 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.521 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.521 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.521 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.521 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.521 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.521 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.522 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.541 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.541 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.541 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.541 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.541 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.541 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.542 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.561 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.561 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.561 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.561 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.561 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.561 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.562 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.581 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.581 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.581 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.581 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.581 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.581 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.621 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.621 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.621 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.621 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.621 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.621 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.622 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.679 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.680 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.680 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.680 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.680 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.680 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.680 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.701 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.701 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.701 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.701 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.701 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.701 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.701 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.702 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.702 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@886768013(state=PENDING, result=null, callback=(continuation of Deferred@892178935 after retry RPC after error@1614931077) -> (continuation of Deferred@406930092 after retry RPC after error@1954217792) -> (continuation of Deferred@763456017 after retry RPC after error@1108072168) -> (continuation of Deferred@1347266599 after retry RPC after error@1945034129) -> (continuation of Deferred@1436147743 after retry RPC after error@1228525778) -> (continuation of Deferred@1946703170 after retry RPC after error@1896815623) -> (continuation of Deferred@1804370721 after retry RPC after error@1815372226), errback=(continuation of Deferred@892178935 after retry RPC after error@1614931077) -> (continuation of Deferred@406930092 after retry RPC after error@1954217792) -> (continuation of Deferred@763456017 after retry RPC after error@1108072168) -> (continuation of Deferred@1347266599 after retry RPC after error@1945034129) -> (continuation of Deferred@1436147743 after retry RPC after error@1228525778) -> (continuation of Deferred@1946703170 after retry RPC after error@1896815623) -> (continuation of Deferred@1804370721 after retry RPC after error@1815372226)))
03:00:03.702 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@886768013(state=PENDING, result=null, callback=(continuation of Deferred@892178935 after retry RPC after error@1614931077) -> (continuation of Deferred@406930092 after retry RPC after error@1954217792) -> (continuation of Deferred@763456017 after retry RPC after error@1108072168) -> (continuation of Deferred@1347266599 after retry RPC after error@1945034129) -> (continuation of Deferred@1436147743 after retry RPC after error@1228525778) -> (continuation of Deferred@1946703170 after retry RPC after error@1896815623) -> (continuation of Deferred@1804370721 after retry RPC after error@1815372226), errback=(continuation of Deferred@892178935 after retry RPC after error@1614931077) -> (continuation of Deferred@406930092 after retry RPC after error@1954217792) -> (continuation of Deferred@763456017 after retry RPC after error@1108072168) -> (continuation of Deferred@1347266599 after retry RPC after error@1945034129) -> (continuation of Deferred@1436147743 after retry RPC after error@1228525778) -> (continuation of Deferred@1946703170 after retry RPC after error@1896815623) -> (continuation of Deferred@1804370721 after retry RPC after error@1815372226)))
03:00:03.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.721 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.721 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.721 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.721 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.721 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.721 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.721 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.722 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.741 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.741 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.741 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.741 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.741 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.741 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.742 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.761 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.761 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.761 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.761 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.761 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.761 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.781 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.781 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.781 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.781 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.781 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.781 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.800 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.800 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.801 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.801 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.801 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.801 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.841 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.841 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.841 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.841 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.841 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.841 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.860 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.861 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.861 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.861 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.861 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.861 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.901 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.901 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.901 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.901 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.901 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.901 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.902 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.902 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1301111931(state=PENDING, result=null, callback=(continuation of Deferred@754157973 after retry RPC after error@317608066) -> (continuation of Deferred@1231186670 after retry RPC after error@1877635333) -> (continuation of Deferred@1600512588 after retry RPC after error@1064289122) -> (continuation of Deferred@1694559099 after retry RPC after error@1499389063) -> (continuation of Deferred@693707362 after retry RPC after error@1751941368) -> (continuation of Deferred@173401533 after retry RPC after error@624999060) -> (continuation of Deferred@1173018220 after retry RPC after error@1234040169), errback=(continuation of Deferred@754157973 after retry RPC after error@317608066) -> (continuation of Deferred@1231186670 after retry RPC after error@1877635333) -> (continuation of Deferred@1600512588 after retry RPC after error@1064289122) -> (continuation of Deferred@1694559099 after retry RPC after error@1499389063) -> (continuation of Deferred@693707362 after retry RPC after error@1751941368) -> (continuation of Deferred@173401533 after retry RPC after error@624999060) -> (continuation of Deferred@1173018220 after retry RPC after error@1234040169)))
03:00:03.902 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1301111931(state=PENDING, result=null, callback=(continuation of Deferred@754157973 after retry RPC after error@317608066) -> (continuation of Deferred@1231186670 after retry RPC after error@1877635333) -> (continuation of Deferred@1600512588 after retry RPC after error@1064289122) -> (continuation of Deferred@1694559099 after retry RPC after error@1499389063) -> (continuation of Deferred@693707362 after retry RPC after error@1751941368) -> (continuation of Deferred@173401533 after retry RPC after error@624999060) -> (continuation of Deferred@1173018220 after retry RPC after error@1234040169), errback=(continuation of Deferred@754157973 after retry RPC after error@317608066) -> (continuation of Deferred@1231186670 after retry RPC after error@1877635333) -> (continuation of Deferred@1600512588 after retry RPC after error@1064289122) -> (continuation of Deferred@1694559099 after retry RPC after error@1499389063) -> (continuation of Deferred@693707362 after retry RPC after error@1751941368) -> (continuation of Deferred@173401533 after retry RPC after error@624999060) -> (continuation of Deferred@1173018220 after retry RPC after error@1234040169)))
03:00:03.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.921 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.921 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.921 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.921 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.921 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.921 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.922 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.941 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.941 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.941 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.941 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.941 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.941 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.942 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.960 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.961 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.961 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.961 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.961 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.961 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:03.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:03.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:03.980 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.980 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.980 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:03.980 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.981 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.981 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:03.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.000 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.000 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.000 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.000 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.000 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.000 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.040 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.040 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.040 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.040 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.040 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.041 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.041 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.080 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.080 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.080 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.081 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.081 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.081 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.081 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.161 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.161 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.161 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.161 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.161 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.161 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.161 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.162 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.180 [DEBUG - pool-40-thread-1] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1181208674(state=PENDING, result=null, callback=(continuation of Deferred@1018556304 after retry RPC after error@1457069270) -> (continuation of Deferred@124543503 after retry RPC after error@2146457152) -> (continuation of Deferred@1814753089 after retry RPC after error@1921737150) -> (continuation of Deferred@370346490 after retry RPC after error@1798049582) -> (continuation of Deferred@1355261421 after retry RPC after error@2036906202) -> (continuation of Deferred@2000302804 after retry RPC after error@2095322722) -> (continuation of Deferred@291581150 after retry RPC after error@770963520) -> (continuation of Deferred@200922040 after retry RPC after error@1467122595), errback=(continuation of Deferred@1018556304 after retry RPC after error@1457069270) -> (continuation of Deferred@124543503 after retry RPC after error@2146457152) -> (continuation of Deferred@1814753089 after retry RPC after error@1921737150) -> (continuation of Deferred@370346490 after retry RPC after error@1798049582) -> (continuation of Deferred@1355261421 after retry RPC after error@2036906202) -> (continuation of Deferred@2000302804 after retry RPC after error@2095322722) -> (continuation of Deferred@291581150 after retry RPC after error@770963520) -> (continuation of Deferred@200922040 after retry RPC after error@1467122595)))
03:00:04.181 [WARN - pool-40-thread-1] (HashedWheelTimer.java:729) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
03:00:04.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.200 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.201 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.201 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.201 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.201 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.201 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.221 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.221 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.221 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.221 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.221 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.221 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.241 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.241 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.241 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.242 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.242 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.242 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.242 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.260 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.260 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.260 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.260 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.260 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.261 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.261 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.280 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.280 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.280 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.280 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.280 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.281 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.300 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.300 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.300 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.300 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.301 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.301 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.319 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.320 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.320 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.320 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.320 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.320 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.320 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.400 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.400 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.400 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.401 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.401 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.401 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.402 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@51171745(state=PENDING, result=null, callback=(continuation of Deferred@1295341596 after retry RPC after error@882483864) -> (continuation of Deferred@1155292118 after retry RPC after error@565713717) -> (continuation of Deferred@1177178790 after retry RPC after error@1071560089) -> (continuation of Deferred@435837791 after retry RPC after error@2021296870) -> (continuation of Deferred@1825654829 after retry RPC after error@644438611) -> (continuation of Deferred@1837292898 after retry RPC after error@1099126670) -> (continuation of Deferred@662044595 after retry RPC after error@1268945612), errback=(continuation of Deferred@1295341596 after retry RPC after error@882483864) -> (continuation of Deferred@1155292118 after retry RPC after error@565713717) -> (continuation of Deferred@1177178790 after retry RPC after error@1071560089) -> (continuation of Deferred@435837791 after retry RPC after error@2021296870) -> (continuation of Deferred@1825654829 after retry RPC after error@644438611) -> (continuation of Deferred@1837292898 after retry RPC after error@1099126670) -> (continuation of Deferred@662044595 after retry RPC after error@1268945612)))
03:00:04.402 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@51171745(state=PENDING, result=null, callback=(continuation of Deferred@1295341596 after retry RPC after error@882483864) -> (continuation of Deferred@1155292118 after retry RPC after error@565713717) -> (continuation of Deferred@1177178790 after retry RPC after error@1071560089) -> (continuation of Deferred@435837791 after retry RPC after error@2021296870) -> (continuation of Deferred@1825654829 after retry RPC after error@644438611) -> (continuation of Deferred@1837292898 after retry RPC after error@1099126670) -> (continuation of Deferred@662044595 after retry RPC after error@1268945612), errback=(continuation of Deferred@1295341596 after retry RPC after error@882483864) -> (continuation of Deferred@1155292118 after retry RPC after error@565713717) -> (continuation of Deferred@1177178790 after retry RPC after error@1071560089) -> (continuation of Deferred@435837791 after retry RPC after error@2021296870) -> (continuation of Deferred@1825654829 after retry RPC after error@644438611) -> (continuation of Deferred@1837292898 after retry RPC after error@1099126670) -> (continuation of Deferred@662044595 after retry RPC after error@1268945612)))
03:00:04.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.420 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.420 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.420 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.420 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.421 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.421 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.421 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.440 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.440 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.440 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.440 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.440 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.440 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.440 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.440 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.460 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.460 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.461 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.460 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.461 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.461 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.461 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.480 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.480 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.480 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.480 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.481 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.481 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.481 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.500 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.500 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.500 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.500 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.501 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.501 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.502 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.540 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.541 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.541 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.541 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.541 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.541 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.541 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.541 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.559 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.560 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.560 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.560 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.560 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.560 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.561 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.561 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.620 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.620 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.621 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.621 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.621 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.621 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.622 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@222740821(state=PENDING, result=null, callback=(continuation of Deferred@1196783796 after retry RPC after error@39608123) -> (continuation of Deferred@1189950267 after retry RPC after error@2098863834) -> (continuation of Deferred@300304661 after retry RPC after error@1898150293) -> (continuation of Deferred@1584986209 after retry RPC after error@1941553594) -> (continuation of Deferred@1354010681 after retry RPC after error@984720491) -> (continuation of Deferred@588511404 after retry RPC after error@513427813) -> (continuation of Deferred@1784362967 after retry RPC after error@1576948747), errback=(continuation of Deferred@1196783796 after retry RPC after error@39608123) -> (continuation of Deferred@1189950267 after retry RPC after error@2098863834) -> (continuation of Deferred@300304661 after retry RPC after error@1898150293) -> (continuation of Deferred@1584986209 after retry RPC after error@1941553594) -> (continuation of Deferred@1354010681 after retry RPC after error@984720491) -> (continuation of Deferred@588511404 after retry RPC after error@513427813) -> (continuation of Deferred@1784362967 after retry RPC after error@1576948747)))
03:00:04.622 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@222740821(state=PENDING, result=null, callback=(continuation of Deferred@1196783796 after retry RPC after error@39608123) -> (continuation of Deferred@1189950267 after retry RPC after error@2098863834) -> (continuation of Deferred@300304661 after retry RPC after error@1898150293) -> (continuation of Deferred@1584986209 after retry RPC after error@1941553594) -> (continuation of Deferred@1354010681 after retry RPC after error@984720491) -> (continuation of Deferred@588511404 after retry RPC after error@513427813) -> (continuation of Deferred@1784362967 after retry RPC after error@1576948747), errback=(continuation of Deferred@1196783796 after retry RPC after error@39608123) -> (continuation of Deferred@1189950267 after retry RPC after error@2098863834) -> (continuation of Deferred@300304661 after retry RPC after error@1898150293) -> (continuation of Deferred@1584986209 after retry RPC after error@1941553594) -> (continuation of Deferred@1354010681 after retry RPC after error@984720491) -> (continuation of Deferred@588511404 after retry RPC after error@513427813) -> (continuation of Deferred@1784362967 after retry RPC after error@1576948747)))
03:00:04.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.640 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.640 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.640 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.640 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.641 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.641 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.660 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.660 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.660 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.660 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.660 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.661 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.681 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.681 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.681 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.681 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.681 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.681 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.682 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.701 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.701 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.701 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.701 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.701 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.701 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.702 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.721 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.721 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.721 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.721 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.721 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.721 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.721 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.761 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.761 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.761 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.761 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.761 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.761 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.761 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.762 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.781 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.781 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.781 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.781 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.781 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.781 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.781 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.782 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.860 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.861 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.861 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.861 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.861 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.861 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.862 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1886850087(state=PENDING, result=null, callback=(continuation of Deferred@284388114 after retry RPC after error@1885212409) -> (continuation of Deferred@891894138 after retry RPC after error@2104954316) -> (continuation of Deferred@1993818763 after retry RPC after error@1996403566) -> (continuation of Deferred@182494754 after retry RPC after error@1337775071) -> (continuation of Deferred@53674937 after retry RPC after error@5612914) -> (continuation of Deferred@260817644 after retry RPC after error@637586153) -> (continuation of Deferred@1035287387 after retry RPC after error@275150762), errback=(continuation of Deferred@284388114 after retry RPC after error@1885212409) -> (continuation of Deferred@891894138 after retry RPC after error@2104954316) -> (continuation of Deferred@1993818763 after retry RPC after error@1996403566) -> (continuation of Deferred@182494754 after retry RPC after error@1337775071) -> (continuation of Deferred@53674937 after retry RPC after error@5612914) -> (continuation of Deferred@260817644 after retry RPC after error@637586153) -> (continuation of Deferred@1035287387 after retry RPC after error@275150762)))
03:00:04.862 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1886850087(state=PENDING, result=null, callback=(continuation of Deferred@284388114 after retry RPC after error@1885212409) -> (continuation of Deferred@891894138 after retry RPC after error@2104954316) -> (continuation of Deferred@1993818763 after retry RPC after error@1996403566) -> (continuation of Deferred@182494754 after retry RPC after error@1337775071) -> (continuation of Deferred@53674937 after retry RPC after error@5612914) -> (continuation of Deferred@260817644 after retry RPC after error@637586153) -> (continuation of Deferred@1035287387 after retry RPC after error@275150762), errback=(continuation of Deferred@284388114 after retry RPC after error@1885212409) -> (continuation of Deferred@891894138 after retry RPC after error@2104954316) -> (continuation of Deferred@1993818763 after retry RPC after error@1996403566) -> (continuation of Deferred@182494754 after retry RPC after error@1337775071) -> (continuation of Deferred@53674937 after retry RPC after error@5612914) -> (continuation of Deferred@260817644 after retry RPC after error@637586153) -> (continuation of Deferred@1035287387 after retry RPC after error@275150762)))
03:00:04.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.880 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.880 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.881 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.881 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.881 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.881 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.900 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.900 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.900 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.900 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.900 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.901 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.919 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.920 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.920 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.920 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.920 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.920 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.920 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.939 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.939 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.940 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.940 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.940 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.940 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.940 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.940 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.961 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.961 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.961 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.961 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.961 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.961 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:04.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:04.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:04.981 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.981 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.981 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.981 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.981 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:04.981 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:04.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.041 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.041 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.041 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.041 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.041 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.041 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.041 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.042 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.042 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [163ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1214007042(state=PENDING, result=null, callback=(continuation of Deferred@92759570 after retry RPC after error@743513514) -> (continuation of Deferred@1705087013 after retry RPC after error@1332719556) -> (continuation of Deferred@1902573517 after retry RPC after error@1434322950) -> (continuation of Deferred@1010272820 after retry RPC after error@244581369) -> (continuation of Deferred@1208632697 after retry RPC after error@1829849670) -> (continuation of Deferred@1836518394 after retry RPC after error@1129465446), errback=(continuation of Deferred@92759570 after retry RPC after error@743513514) -> (continuation of Deferred@1705087013 after retry RPC after error@1332719556) -> (continuation of Deferred@1902573517 after retry RPC after error@1434322950) -> (continuation of Deferred@1010272820 after retry RPC after error@244581369) -> (continuation of Deferred@1208632697 after retry RPC after error@1829849670) -> (continuation of Deferred@1836518394 after retry RPC after error@1129465446)))
03:00:05.042 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [163ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1214007042(state=PENDING, result=null, callback=(continuation of Deferred@92759570 after retry RPC after error@743513514) -> (continuation of Deferred@1705087013 after retry RPC after error@1332719556) -> (continuation of Deferred@1902573517 after retry RPC after error@1434322950) -> (continuation of Deferred@1010272820 after retry RPC after error@244581369) -> (continuation of Deferred@1208632697 after retry RPC after error@1829849670) -> (continuation of Deferred@1836518394 after retry RPC after error@1129465446), errback=(continuation of Deferred@92759570 after retry RPC after error@743513514) -> (continuation of Deferred@1705087013 after retry RPC after error@1332719556) -> (continuation of Deferred@1902573517 after retry RPC after error@1434322950) -> (continuation of Deferred@1010272820 after retry RPC after error@244581369) -> (continuation of Deferred@1208632697 after retry RPC after error@1829849670) -> (continuation of Deferred@1836518394 after retry RPC after error@1129465446)))
03:00:05.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.061 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.061 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.061 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.061 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.061 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.061 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.062 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.081 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.081 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.081 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.081 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.081 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.081 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.081 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.101 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.101 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.101 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.101 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.101 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.101 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.121 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.121 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.121 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.121 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.121 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.121 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.122 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.141 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.141 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.141 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.141 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.141 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.141 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.141 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.160 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.160 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.161 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.161 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.161 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.161 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.201 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.201 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.201 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.201 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.201 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.201 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.202 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.299 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.300 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.300 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.300 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.301 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.301 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.301 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.301 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@745076205(state=PENDING, result=null, callback=(continuation of Deferred@2087854609 after retry RPC after error@369660194) -> (continuation of Deferred@767620754 after retry RPC after error@1181530458) -> (continuation of Deferred@736323884 after retry RPC after error@2033216755) -> (continuation of Deferred@1408135457 after retry RPC after error@1894491247) -> (continuation of Deferred@1081149783 after retry RPC after error@987538014) -> (continuation of Deferred@438020957 after retry RPC after error@939441325) -> (continuation of Deferred@2131570868 after retry RPC after error@547140441), errback=(continuation of Deferred@2087854609 after retry RPC after error@369660194) -> (continuation of Deferred@767620754 after retry RPC after error@1181530458) -> (continuation of Deferred@736323884 after retry RPC after error@2033216755) -> (continuation of Deferred@1408135457 after retry RPC after error@1894491247) -> (continuation of Deferred@1081149783 after retry RPC after error@987538014) -> (continuation of Deferred@438020957 after retry RPC after error@939441325) -> (continuation of Deferred@2131570868 after retry RPC after error@547140441)))
03:00:05.302 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@745076205(state=PENDING, result=null, callback=(continuation of Deferred@2087854609 after retry RPC after error@369660194) -> (continuation of Deferred@767620754 after retry RPC after error@1181530458) -> (continuation of Deferred@736323884 after retry RPC after error@2033216755) -> (continuation of Deferred@1408135457 after retry RPC after error@1894491247) -> (continuation of Deferred@1081149783 after retry RPC after error@987538014) -> (continuation of Deferred@438020957 after retry RPC after error@939441325) -> (continuation of Deferred@2131570868 after retry RPC after error@547140441), errback=(continuation of Deferred@2087854609 after retry RPC after error@369660194) -> (continuation of Deferred@767620754 after retry RPC after error@1181530458) -> (continuation of Deferred@736323884 after retry RPC after error@2033216755) -> (continuation of Deferred@1408135457 after retry RPC after error@1894491247) -> (continuation of Deferred@1081149783 after retry RPC after error@987538014) -> (continuation of Deferred@438020957 after retry RPC after error@939441325) -> (continuation of Deferred@2131570868 after retry RPC after error@547140441)))
03:00:05.319 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.320 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.320 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.320 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.320 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.320 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.320 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.339 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.339 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.340 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.340 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.340 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.340 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.340 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.340 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.360 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.360 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.360 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.361 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.361 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.361 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.362 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.379 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.380 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.380 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.380 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.380 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.380 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.380 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.401 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.401 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.401 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.401 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.401 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.401 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.402 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.439 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.440 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.440 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.440 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.440 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.440 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.440 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.500 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.501 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.501 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.501 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.501 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.501 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.502 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.502 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@870148690(state=PENDING, result=null, callback=(continuation of Deferred@1990070539 after retry RPC after error@1142696585) -> (continuation of Deferred@2052575698 after retry RPC after error@570469168) -> (continuation of Deferred@613474354 after retry RPC after error@33410635) -> (continuation of Deferred@1172330215 after retry RPC after error@1646148672) -> (continuation of Deferred@1093456193 after retry RPC after error@776786105) -> (continuation of Deferred@1776086265 after retry RPC after error@2058564030), errback=(continuation of Deferred@1990070539 after retry RPC after error@1142696585) -> (continuation of Deferred@2052575698 after retry RPC after error@570469168) -> (continuation of Deferred@613474354 after retry RPC after error@33410635) -> (continuation of Deferred@1172330215 after retry RPC after error@1646148672) -> (continuation of Deferred@1093456193 after retry RPC after error@776786105) -> (continuation of Deferred@1776086265 after retry RPC after error@2058564030)))
03:00:05.502 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@870148690(state=PENDING, result=null, callback=(continuation of Deferred@1990070539 after retry RPC after error@1142696585) -> (continuation of Deferred@2052575698 after retry RPC after error@570469168) -> (continuation of Deferred@613474354 after retry RPC after error@33410635) -> (continuation of Deferred@1172330215 after retry RPC after error@1646148672) -> (continuation of Deferred@1093456193 after retry RPC after error@776786105) -> (continuation of Deferred@1776086265 after retry RPC after error@2058564030), errback=(continuation of Deferred@1990070539 after retry RPC after error@1142696585) -> (continuation of Deferred@2052575698 after retry RPC after error@570469168) -> (continuation of Deferred@613474354 after retry RPC after error@33410635) -> (continuation of Deferred@1172330215 after retry RPC after error@1646148672) -> (continuation of Deferred@1093456193 after retry RPC after error@776786105) -> (continuation of Deferred@1776086265 after retry RPC after error@2058564030)))
03:00:05.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.520 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.521 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.521 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.521 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.521 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.521 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.522 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.540 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.541 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.541 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.541 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.541 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.541 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.542 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.560 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.560 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.560 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.561 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.561 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.561 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.561 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.581 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.581 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.581 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.581 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.581 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.581 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.582 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.600 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.601 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.601 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.601 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.601 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.601 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.602 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.641 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.641 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.641 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.641 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.641 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.641 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.642 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.681 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.681 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.681 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.681 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.681 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.681 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.682 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.721 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.721 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.721 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.721 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.721 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.721 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.721 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.722 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.761 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.761 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.761 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.761 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.761 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.761 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.762 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@847084656(state=PENDING, result=null, callback=(continuation of Deferred@1768353603 after retry RPC after error@905063143) -> (continuation of Deferred@679677885 after retry RPC after error@1908697121) -> (continuation of Deferred@797351341 after retry RPC after error@193413032) -> (continuation of Deferred@994649986 after retry RPC after error@1763624362) -> (continuation of Deferred@1928218321 after retry RPC after error@1084640603) -> (continuation of Deferred@849414138 after retry RPC after error@764258193) -> (continuation of Deferred@760591423 after retry RPC after error@1609295963), errback=(continuation of Deferred@1768353603 after retry RPC after error@905063143) -> (continuation of Deferred@679677885 after retry RPC after error@1908697121) -> (continuation of Deferred@797351341 after retry RPC after error@193413032) -> (continuation of Deferred@994649986 after retry RPC after error@1763624362) -> (continuation of Deferred@1928218321 after retry RPC after error@1084640603) -> (continuation of Deferred@849414138 after retry RPC after error@764258193) -> (continuation of Deferred@760591423 after retry RPC after error@1609295963)))
03:00:05.762 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@847084656(state=PENDING, result=null, callback=(continuation of Deferred@1768353603 after retry RPC after error@905063143) -> (continuation of Deferred@679677885 after retry RPC after error@1908697121) -> (continuation of Deferred@797351341 after retry RPC after error@193413032) -> (continuation of Deferred@994649986 after retry RPC after error@1763624362) -> (continuation of Deferred@1928218321 after retry RPC after error@1084640603) -> (continuation of Deferred@849414138 after retry RPC after error@764258193) -> (continuation of Deferred@760591423 after retry RPC after error@1609295963), errback=(continuation of Deferred@1768353603 after retry RPC after error@905063143) -> (continuation of Deferred@679677885 after retry RPC after error@1908697121) -> (continuation of Deferred@797351341 after retry RPC after error@193413032) -> (continuation of Deferred@994649986 after retry RPC after error@1763624362) -> (continuation of Deferred@1928218321 after retry RPC after error@1084640603) -> (continuation of Deferred@849414138 after retry RPC after error@764258193) -> (continuation of Deferred@760591423 after retry RPC after error@1609295963)))
03:00:05.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.781 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.781 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.781 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.781 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.781 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.781 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.782 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.801 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.801 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.801 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.801 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.801 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.801 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.821 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.821 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.821 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.821 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.821 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.821 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.841 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.841 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.841 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.841 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.841 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.841 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.860 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.860 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.860 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.860 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.861 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.861 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.880 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.880 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.880 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.881 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.881 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.881 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.900 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.900 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.900 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.901 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.901 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.901 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.960 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.960 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.960 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.961 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.961 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.961 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.961 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1029764191(state=PENDING, result=null, callback=(continuation of Deferred@1775124919 after retry RPC after error@416800914) -> (continuation of Deferred@625313115 after retry RPC after error@524111776) -> (continuation of Deferred@82142729 after retry RPC after error@588371319) -> (continuation of Deferred@712504083 after retry RPC after error@1598220722) -> (continuation of Deferred@1882720637 after retry RPC after error@1571803232) -> (continuation of Deferred@1213554605 after retry RPC after error@9576556) -> (continuation of Deferred@810656482 after retry RPC after error@324950104), errback=(continuation of Deferred@1775124919 after retry RPC after error@416800914) -> (continuation of Deferred@625313115 after retry RPC after error@524111776) -> (continuation of Deferred@82142729 after retry RPC after error@588371319) -> (continuation of Deferred@712504083 after retry RPC after error@1598220722) -> (continuation of Deferred@1882720637 after retry RPC after error@1571803232) -> (continuation of Deferred@1213554605 after retry RPC after error@9576556) -> (continuation of Deferred@810656482 after retry RPC after error@324950104)))
03:00:05.962 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1029764191(state=PENDING, result=null, callback=(continuation of Deferred@1775124919 after retry RPC after error@416800914) -> (continuation of Deferred@625313115 after retry RPC after error@524111776) -> (continuation of Deferred@82142729 after retry RPC after error@588371319) -> (continuation of Deferred@712504083 after retry RPC after error@1598220722) -> (continuation of Deferred@1882720637 after retry RPC after error@1571803232) -> (continuation of Deferred@1213554605 after retry RPC after error@9576556) -> (continuation of Deferred@810656482 after retry RPC after error@324950104), errback=(continuation of Deferred@1775124919 after retry RPC after error@416800914) -> (continuation of Deferred@625313115 after retry RPC after error@524111776) -> (continuation of Deferred@82142729 after retry RPC after error@588371319) -> (continuation of Deferred@712504083 after retry RPC after error@1598220722) -> (continuation of Deferred@1882720637 after retry RPC after error@1571803232) -> (continuation of Deferred@1213554605 after retry RPC after error@9576556) -> (continuation of Deferred@810656482 after retry RPC after error@324950104)))
03:00:05.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:05.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:05.980 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.980 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.980 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:05.980 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.980 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.980 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:05.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:05.999 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.000 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.000 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.000 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.000 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.000 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.001 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.020 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.020 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.020 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.021 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.021 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.021 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.040 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.040 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.040 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.040 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.040 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.040 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.041 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.059 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.060 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.060 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.060 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.060 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.060 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.060 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.062 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.100 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.100 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.100 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.100 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.101 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.101 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.102 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.160 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.161 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.161 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.161 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.161 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.161 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.162 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1567613669(state=PENDING, result=null, callback=(continuation of Deferred@1007873416 after retry RPC after error@1803308590) -> (continuation of Deferred@738042068 after retry RPC after error@161941380) -> (continuation of Deferred@586335637 after retry RPC after error@737600219) -> (continuation of Deferred@1833634929 after retry RPC after error@1549905275) -> (continuation of Deferred@1753945359 after retry RPC after error@1954005759) -> (continuation of Deferred@1105879172 after retry RPC after error@280037303), errback=(continuation of Deferred@1007873416 after retry RPC after error@1803308590) -> (continuation of Deferred@738042068 after retry RPC after error@161941380) -> (continuation of Deferred@586335637 after retry RPC after error@737600219) -> (continuation of Deferred@1833634929 after retry RPC after error@1549905275) -> (continuation of Deferred@1753945359 after retry RPC after error@1954005759) -> (continuation of Deferred@1105879172 after retry RPC after error@280037303)))
03:00:06.162 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1567613669(state=PENDING, result=null, callback=(continuation of Deferred@1007873416 after retry RPC after error@1803308590) -> (continuation of Deferred@738042068 after retry RPC after error@161941380) -> (continuation of Deferred@586335637 after retry RPC after error@737600219) -> (continuation of Deferred@1833634929 after retry RPC after error@1549905275) -> (continuation of Deferred@1753945359 after retry RPC after error@1954005759) -> (continuation of Deferred@1105879172 after retry RPC after error@280037303), errback=(continuation of Deferred@1007873416 after retry RPC after error@1803308590) -> (continuation of Deferred@738042068 after retry RPC after error@161941380) -> (continuation of Deferred@586335637 after retry RPC after error@737600219) -> (continuation of Deferred@1833634929 after retry RPC after error@1549905275) -> (continuation of Deferred@1753945359 after retry RPC after error@1954005759) -> (continuation of Deferred@1105879172 after retry RPC after error@280037303)))
03:00:06.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.180 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.180 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.180 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.180 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.181 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.181 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.200 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.200 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.200 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.200 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.200 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.201 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.219 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.220 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.220 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.220 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.220 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.220 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.220 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.239 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.239 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.240 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.240 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.240 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.240 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.240 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.240 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.241 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.260 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.261 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.261 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.261 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.261 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.261 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.262 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.281 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.281 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.281 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.281 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.281 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.281 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.341 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.341 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.341 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.341 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.341 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.341 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.399 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.400 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.400 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.400 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.400 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.400 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.400 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.439 [DEBUG - pool-40-thread-1] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@956498311(state=PENDING, result=null, callback=(continuation of Deferred@434205529 after retry RPC after error@1916132606) -> (continuation of Deferred@1788385926 after retry RPC after error@1270260075) -> (continuation of Deferred@1507813631 after retry RPC after error@89234710) -> (continuation of Deferred@733182864 after retry RPC after error@1441802868) -> (continuation of Deferred@2009737381 after retry RPC after error@2130354791) -> (continuation of Deferred@1319451091 after retry RPC after error@449976825) -> (continuation of Deferred@1023116550 after retry RPC after error@398061924) -> (continuation of Deferred@1890920929 after retry RPC after error@2057501663), errback=(continuation of Deferred@434205529 after retry RPC after error@1916132606) -> (continuation of Deferred@1788385926 after retry RPC after error@1270260075) -> (continuation of Deferred@1507813631 after retry RPC after error@89234710) -> (continuation of Deferred@733182864 after retry RPC after error@1441802868) -> (continuation of Deferred@2009737381 after retry RPC after error@2130354791) -> (continuation of Deferred@1319451091 after retry RPC after error@449976825) -> (continuation of Deferred@1023116550 after retry RPC after error@398061924) -> (continuation of Deferred@1890920929 after retry RPC after error@2057501663)))
03:00:06.440 [WARN - pool-40-thread-1] (HashedWheelTimer.java:729) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
03:00:06.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.461 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.461 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.461 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.461 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.461 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.462 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.462 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.480 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.480 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.481 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.481 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.481 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.481 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.482 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.500 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.500 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.500 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.500 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.500 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.500 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.519 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.520 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.520 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.520 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.520 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.521 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.521 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.540 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.540 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.540 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.540 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.540 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.540 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.580 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.580 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.580 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.580 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.580 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.580 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.641 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.641 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.641 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.641 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.641 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.642 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.642 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.701 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.701 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.701 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.701 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.701 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.701 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.702 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.702 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1713920663(state=PENDING, result=null, callback=(continuation of Deferred@979778275 after retry RPC after error@1426745658) -> (continuation of Deferred@1606607912 after retry RPC after error@2073747877) -> (continuation of Deferred@1889565975 after retry RPC after error@1994758528) -> (continuation of Deferred@1053737168 after retry RPC after error@318085477) -> (continuation of Deferred@1632503529 after retry RPC after error@1002289881) -> (continuation of Deferred@1933387070 after retry RPC after error@1295889549) -> (continuation of Deferred@1549113997 after retry RPC after error@174626988), errback=(continuation of Deferred@979778275 after retry RPC after error@1426745658) -> (continuation of Deferred@1606607912 after retry RPC after error@2073747877) -> (continuation of Deferred@1889565975 after retry RPC after error@1994758528) -> (continuation of Deferred@1053737168 after retry RPC after error@318085477) -> (continuation of Deferred@1632503529 after retry RPC after error@1002289881) -> (continuation of Deferred@1933387070 after retry RPC after error@1295889549) -> (continuation of Deferred@1549113997 after retry RPC after error@174626988)))
03:00:06.702 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1713920663(state=PENDING, result=null, callback=(continuation of Deferred@979778275 after retry RPC after error@1426745658) -> (continuation of Deferred@1606607912 after retry RPC after error@2073747877) -> (continuation of Deferred@1889565975 after retry RPC after error@1994758528) -> (continuation of Deferred@1053737168 after retry RPC after error@318085477) -> (continuation of Deferred@1632503529 after retry RPC after error@1002289881) -> (continuation of Deferred@1933387070 after retry RPC after error@1295889549) -> (continuation of Deferred@1549113997 after retry RPC after error@174626988), errback=(continuation of Deferred@979778275 after retry RPC after error@1426745658) -> (continuation of Deferred@1606607912 after retry RPC after error@2073747877) -> (continuation of Deferred@1889565975 after retry RPC after error@1994758528) -> (continuation of Deferred@1053737168 after retry RPC after error@318085477) -> (continuation of Deferred@1632503529 after retry RPC after error@1002289881) -> (continuation of Deferred@1933387070 after retry RPC after error@1295889549) -> (continuation of Deferred@1549113997 after retry RPC after error@174626988)))
03:00:06.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.720 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.720 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.721 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.721 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.721 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.721 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.721 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.740 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.740 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.740 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.741 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.741 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.741 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.760 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.760 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.760 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.761 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.761 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.761 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.779 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.780 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.780 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.780 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.780 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.780 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.780 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.799 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.800 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.800 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.800 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.800 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.800 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.800 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.821 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.821 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.821 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.821 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.821 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.821 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.822 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:06.831426 21331 raft_consensus.cc:534] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 56099a6022da440a81f20e6fe8903fb0)
03:00:06.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:06.831697 21331 raft_consensus.cc:556] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
03:00:06.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:06.832597 21331 leader_election.cc:290] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
03:00:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 03:00:06.833683 20381 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111) [suppressed 3 similar messages]
03:00:06.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 03:00:06.835089 20381 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
03:00:06.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 03:00:06.835251 20378 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111): Network error: Client connection negotiation failed: client connection to 127.19.203.254:46111: connect: Connection refused (error 111)
03:00:06.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:06.835417 20378 leader_election.cc:304] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c; no voters: 56099a6022da440a81f20e6fe8903fb0, efb4c69b20bd4f9692d496c33ddbb24e
03:00:06.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:06.835733 21331 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
03:00:06.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.861 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.861 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.861 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.861 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.861 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.861 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.861 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.862 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.919 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.920 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.920 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.920 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.920 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.920 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.920 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.921 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@751372473(state=PENDING, result=null, callback=(continuation of Deferred@1688510416 after retry RPC after error@1474862741) -> (continuation of Deferred@813253933 after retry RPC after error@1410404187) -> (continuation of Deferred@344674326 after retry RPC after error@126122020) -> (continuation of Deferred@2100168148 after retry RPC after error@555329277) -> (continuation of Deferred@1868323112 after retry RPC after error@1262311871) -> (continuation of Deferred@1185998249 after retry RPC after error@720539084) -> (continuation of Deferred@661259427 after retry RPC after error@112939344), errback=(continuation of Deferred@1688510416 after retry RPC after error@1474862741) -> (continuation of Deferred@813253933 after retry RPC after error@1410404187) -> (continuation of Deferred@344674326 after retry RPC after error@126122020) -> (continuation of Deferred@2100168148 after retry RPC after error@555329277) -> (continuation of Deferred@1868323112 after retry RPC after error@1262311871) -> (continuation of Deferred@1185998249 after retry RPC after error@720539084) -> (continuation of Deferred@661259427 after retry RPC after error@112939344)))
03:00:06.922 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@751372473(state=PENDING, result=null, callback=(continuation of Deferred@1688510416 after retry RPC after error@1474862741) -> (continuation of Deferred@813253933 after retry RPC after error@1410404187) -> (continuation of Deferred@344674326 after retry RPC after error@126122020) -> (continuation of Deferred@2100168148 after retry RPC after error@555329277) -> (continuation of Deferred@1868323112 after retry RPC after error@1262311871) -> (continuation of Deferred@1185998249 after retry RPC after error@720539084) -> (continuation of Deferred@661259427 after retry RPC after error@112939344), errback=(continuation of Deferred@1688510416 after retry RPC after error@1474862741) -> (continuation of Deferred@813253933 after retry RPC after error@1410404187) -> (continuation of Deferred@344674326 after retry RPC after error@126122020) -> (continuation of Deferred@2100168148 after retry RPC after error@555329277) -> (continuation of Deferred@1868323112 after retry RPC after error@1262311871) -> (continuation of Deferred@1185998249 after retry RPC after error@720539084) -> (continuation of Deferred@661259427 after retry RPC after error@112939344)))
03:00:06.939 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.940 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.940 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.940 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.940 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.940 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.940 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.959 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.960 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.960 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.960 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.960 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.960 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.960 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:06.979 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:06.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:06.980 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.980 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.980 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:06.980 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.980 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.980 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:06.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.001 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.001 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.001 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.001 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.001 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.001 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.002 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.021 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.021 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.021 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.021 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.021 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.021 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.022 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.040 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.041 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.041 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.041 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.041 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.041 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.042 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.121 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.121 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.121 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.121 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.121 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.121 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.122 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.122 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2144421721(state=PENDING, result=null, callback=(continuation of Deferred@460230630 after retry RPC after error@1283829121) -> (continuation of Deferred@1534894059 after retry RPC after error@778231890) -> (continuation of Deferred@1100762055 after retry RPC after error@546578384) -> (continuation of Deferred@522739694 after retry RPC after error@644959273) -> (continuation of Deferred@93387091 after retry RPC after error@91884718) -> (continuation of Deferred@1492182208 after retry RPC after error@1951198195), errback=(continuation of Deferred@460230630 after retry RPC after error@1283829121) -> (continuation of Deferred@1534894059 after retry RPC after error@778231890) -> (continuation of Deferred@1100762055 after retry RPC after error@546578384) -> (continuation of Deferred@522739694 after retry RPC after error@644959273) -> (continuation of Deferred@93387091 after retry RPC after error@91884718) -> (continuation of Deferred@1492182208 after retry RPC after error@1951198195)))
03:00:07.122 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2144421721(state=PENDING, result=null, callback=(continuation of Deferred@460230630 after retry RPC after error@1283829121) -> (continuation of Deferred@1534894059 after retry RPC after error@778231890) -> (continuation of Deferred@1100762055 after retry RPC after error@546578384) -> (continuation of Deferred@522739694 after retry RPC after error@644959273) -> (continuation of Deferred@93387091 after retry RPC after error@91884718) -> (continuation of Deferred@1492182208 after retry RPC after error@1951198195), errback=(continuation of Deferred@460230630 after retry RPC after error@1283829121) -> (continuation of Deferred@1534894059 after retry RPC after error@778231890) -> (continuation of Deferred@1100762055 after retry RPC after error@546578384) -> (continuation of Deferred@522739694 after retry RPC after error@644959273) -> (continuation of Deferred@93387091 after retry RPC after error@91884718) -> (continuation of Deferred@1492182208 after retry RPC after error@1951198195)))
03:00:07.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.141 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.141 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.141 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.141 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.141 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.141 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.142 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.161 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.161 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.161 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.161 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.161 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.161 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.162 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.181 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.181 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.181 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.181 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.181 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.181 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.182 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.200 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.201 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.201 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.201 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.201 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.201 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.220 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.220 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.220 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.220 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.220 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.220 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.239 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.240 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.240 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.240 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.240 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.240 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.240 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.241 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.261 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.261 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.261 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.261 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.261 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.261 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.261 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.262 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.340 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.340 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.340 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.341 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.341 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.341 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.341 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1063329569(state=PENDING, result=null, callback=(continuation of Deferred@1777545062 after retry RPC after error@701263986) -> (continuation of Deferred@2031874116 after retry RPC after error@1312055377) -> (continuation of Deferred@1053621998 after retry RPC after error@1950963365) -> (continuation of Deferred@1391854008 after retry RPC after error@1059964255) -> (continuation of Deferred@1716484058 after retry RPC after error@1477310920) -> (continuation of Deferred@982833328 after retry RPC after error@1807938944) -> (continuation of Deferred@1982774917 after retry RPC after error@1096473137), errback=(continuation of Deferred@1777545062 after retry RPC after error@701263986) -> (continuation of Deferred@2031874116 after retry RPC after error@1312055377) -> (continuation of Deferred@1053621998 after retry RPC after error@1950963365) -> (continuation of Deferred@1391854008 after retry RPC after error@1059964255) -> (continuation of Deferred@1716484058 after retry RPC after error@1477310920) -> (continuation of Deferred@982833328 after retry RPC after error@1807938944) -> (continuation of Deferred@1982774917 after retry RPC after error@1096473137)))
03:00:07.342 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1063329569(state=PENDING, result=null, callback=(continuation of Deferred@1777545062 after retry RPC after error@701263986) -> (continuation of Deferred@2031874116 after retry RPC after error@1312055377) -> (continuation of Deferred@1053621998 after retry RPC after error@1950963365) -> (continuation of Deferred@1391854008 after retry RPC after error@1059964255) -> (continuation of Deferred@1716484058 after retry RPC after error@1477310920) -> (continuation of Deferred@982833328 after retry RPC after error@1807938944) -> (continuation of Deferred@1982774917 after retry RPC after error@1096473137), errback=(continuation of Deferred@1777545062 after retry RPC after error@701263986) -> (continuation of Deferred@2031874116 after retry RPC after error@1312055377) -> (continuation of Deferred@1053621998 after retry RPC after error@1950963365) -> (continuation of Deferred@1391854008 after retry RPC after error@1059964255) -> (continuation of Deferred@1716484058 after retry RPC after error@1477310920) -> (continuation of Deferred@982833328 after retry RPC after error@1807938944) -> (continuation of Deferred@1982774917 after retry RPC after error@1096473137)))
03:00:07.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.360 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.360 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.360 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.360 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.360 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.360 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.379 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.380 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.380 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.380 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.380 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.380 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.380 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.399 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.400 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.400 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.400 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.400 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.400 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.400 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.421 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.421 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.421 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.421 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.421 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.421 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.421 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.422 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.441 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.441 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.441 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.441 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.441 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.441 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.442 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.461 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.461 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.461 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.461 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.461 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.461 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.465 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.501 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.501 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.501 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.501 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.501 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.501 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.501 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.502 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.600 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.600 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.600 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.601 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.601 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.601 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.602 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [105ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [106ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@523932214(state=PENDING, result=null, callback=(continuation of Deferred@741875409 after retry RPC after error@537303754) -> (continuation of Deferred@452299168 after retry RPC after error@1368724458) -> (continuation of Deferred@1027031455 after retry RPC after error@695426105) -> (continuation of Deferred@536705086 after retry RPC after error@1712202344) -> (continuation of Deferred@1639099535 after retry RPC after error@1377023508) -> (continuation of Deferred@643504352 after retry RPC after error@827421473) -> (continuation of Deferred@1173579600 after retry RPC after error@169855724), errback=(continuation of Deferred@741875409 after retry RPC after error@537303754) -> (continuation of Deferred@452299168 after retry RPC after error@1368724458) -> (continuation of Deferred@1027031455 after retry RPC after error@695426105) -> (continuation of Deferred@536705086 after retry RPC after error@1712202344) -> (continuation of Deferred@1639099535 after retry RPC after error@1377023508) -> (continuation of Deferred@643504352 after retry RPC after error@827421473) -> (continuation of Deferred@1173579600 after retry RPC after error@169855724)))
03:00:07.602 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [105ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [106ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@523932214(state=PENDING, result=null, callback=(continuation of Deferred@741875409 after retry RPC after error@537303754) -> (continuation of Deferred@452299168 after retry RPC after error@1368724458) -> (continuation of Deferred@1027031455 after retry RPC after error@695426105) -> (continuation of Deferred@536705086 after retry RPC after error@1712202344) -> (continuation of Deferred@1639099535 after retry RPC after error@1377023508) -> (continuation of Deferred@643504352 after retry RPC after error@827421473) -> (continuation of Deferred@1173579600 after retry RPC after error@169855724), errback=(continuation of Deferred@741875409 after retry RPC after error@537303754) -> (continuation of Deferred@452299168 after retry RPC after error@1368724458) -> (continuation of Deferred@1027031455 after retry RPC after error@695426105) -> (continuation of Deferred@536705086 after retry RPC after error@1712202344) -> (continuation of Deferred@1639099535 after retry RPC after error@1377023508) -> (continuation of Deferred@643504352 after retry RPC after error@827421473) -> (continuation of Deferred@1173579600 after retry RPC after error@169855724)))
03:00:07.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.620 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.620 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.620 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.620 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.620 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.621 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.639 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.640 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.640 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.640 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.640 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.640 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.640 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.659 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.660 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.660 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.660 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.660 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.660 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.660 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.681 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.681 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.681 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.681 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.681 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.681 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.682 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.701 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.701 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.701 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.701 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.701 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.701 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.702 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.741 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.741 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.741 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.741 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.741 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.741 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.742 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.799 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.800 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.800 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.800 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.800 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.800 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.800 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.801 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2059258261(state=PENDING, result=null, callback=(continuation of Deferred@1268199218 after retry RPC after error@494302273) -> (continuation of Deferred@1488634851 after retry RPC after error@1281312768) -> (continuation of Deferred@1044578310 after retry RPC after error@272216051) -> (continuation of Deferred@512934860 after retry RPC after error@683378518) -> (continuation of Deferred@249487735 after retry RPC after error@1405452648) -> (continuation of Deferred@1531032703 after retry RPC after error@800332150), errback=(continuation of Deferred@1268199218 after retry RPC after error@494302273) -> (continuation of Deferred@1488634851 after retry RPC after error@1281312768) -> (continuation of Deferred@1044578310 after retry RPC after error@272216051) -> (continuation of Deferred@512934860 after retry RPC after error@683378518) -> (continuation of Deferred@249487735 after retry RPC after error@1405452648) -> (continuation of Deferred@1531032703 after retry RPC after error@800332150)))
03:00:07.802 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2059258261(state=PENDING, result=null, callback=(continuation of Deferred@1268199218 after retry RPC after error@494302273) -> (continuation of Deferred@1488634851 after retry RPC after error@1281312768) -> (continuation of Deferred@1044578310 after retry RPC after error@272216051) -> (continuation of Deferred@512934860 after retry RPC after error@683378518) -> (continuation of Deferred@249487735 after retry RPC after error@1405452648) -> (continuation of Deferred@1531032703 after retry RPC after error@800332150), errback=(continuation of Deferred@1268199218 after retry RPC after error@494302273) -> (continuation of Deferred@1488634851 after retry RPC after error@1281312768) -> (continuation of Deferred@1044578310 after retry RPC after error@272216051) -> (continuation of Deferred@512934860 after retry RPC after error@683378518) -> (continuation of Deferred@249487735 after retry RPC after error@1405452648) -> (continuation of Deferred@1531032703 after retry RPC after error@800332150)))
03:00:07.819 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.820 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.820 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.820 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.820 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.820 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.820 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.841 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.841 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.841 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.841 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.841 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.841 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.860 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.861 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.861 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.861 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.861 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.861 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.880 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.880 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.880 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.881 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.881 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.881 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.900 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.900 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.901 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.901 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.901 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.901 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.902 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:07.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:07.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:07.940 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.940 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.941 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.941 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.941 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:07.941 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:07.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.001 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.001 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.001 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.001 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.001 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.001 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.001 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1650590909(state=PENDING, result=null, callback=(continuation of Deferred@125840000 after retry RPC after error@293437196) -> (continuation of Deferred@238041162 after retry RPC after error@2089128669) -> (continuation of Deferred@2060940263 after retry RPC after error@2680494) -> (continuation of Deferred@257260681 after retry RPC after error@1603667140) -> (continuation of Deferred@505351322 after retry RPC after error@1497732193) -> (continuation of Deferred@2110786292 after retry RPC after error@311394734), errback=(continuation of Deferred@125840000 after retry RPC after error@293437196) -> (continuation of Deferred@238041162 after retry RPC after error@2089128669) -> (continuation of Deferred@2060940263 after retry RPC after error@2680494) -> (continuation of Deferred@257260681 after retry RPC after error@1603667140) -> (continuation of Deferred@505351322 after retry RPC after error@1497732193) -> (continuation of Deferred@2110786292 after retry RPC after error@311394734)))
03:00:08.002 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1650590909(state=PENDING, result=null, callback=(continuation of Deferred@125840000 after retry RPC after error@293437196) -> (continuation of Deferred@238041162 after retry RPC after error@2089128669) -> (continuation of Deferred@2060940263 after retry RPC after error@2680494) -> (continuation of Deferred@257260681 after retry RPC after error@1603667140) -> (continuation of Deferred@505351322 after retry RPC after error@1497732193) -> (continuation of Deferred@2110786292 after retry RPC after error@311394734), errback=(continuation of Deferred@125840000 after retry RPC after error@293437196) -> (continuation of Deferred@238041162 after retry RPC after error@2089128669) -> (continuation of Deferred@2060940263 after retry RPC after error@2680494) -> (continuation of Deferred@257260681 after retry RPC after error@1603667140) -> (continuation of Deferred@505351322 after retry RPC after error@1497732193) -> (continuation of Deferred@2110786292 after retry RPC after error@311394734)))
03:00:08.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.021 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.021 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.021 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.021 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.021 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.021 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.022 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.041 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.041 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.041 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.041 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.041 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.041 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.042 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.060 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.061 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.061 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.061 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.061 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.061 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.062 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.080 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.080 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.080 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.080 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.080 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.080 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.081 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.099 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.100 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.100 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.100 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.100 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.100 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.100 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.139 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.139 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.140 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.140 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.140 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.140 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.140 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.140 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.141 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.199 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.200 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.200 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.200 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.200 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.200 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.200 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.239 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.240 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.240 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.240 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.240 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.240 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.240 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.241 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.241 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2085200289(state=PENDING, result=null, callback=(continuation of Deferred@1534592317 after retry RPC after error@1909375867) -> (continuation of Deferred@739601181 after retry RPC after error@501257111) -> (continuation of Deferred@111815931 after retry RPC after error@33088078) -> (continuation of Deferred@112344303 after retry RPC after error@534215557) -> (continuation of Deferred@1213026672 after retry RPC after error@249639077) -> (continuation of Deferred@430125266 after retry RPC after error@1032616752) -> (continuation of Deferred@535329568 after retry RPC after error@403375200), errback=(continuation of Deferred@1534592317 after retry RPC after error@1909375867) -> (continuation of Deferred@739601181 after retry RPC after error@501257111) -> (continuation of Deferred@111815931 after retry RPC after error@33088078) -> (continuation of Deferred@112344303 after retry RPC after error@534215557) -> (continuation of Deferred@1213026672 after retry RPC after error@249639077) -> (continuation of Deferred@430125266 after retry RPC after error@1032616752) -> (continuation of Deferred@535329568 after retry RPC after error@403375200)))
03:00:08.242 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2085200289(state=PENDING, result=null, callback=(continuation of Deferred@1534592317 after retry RPC after error@1909375867) -> (continuation of Deferred@739601181 after retry RPC after error@501257111) -> (continuation of Deferred@111815931 after retry RPC after error@33088078) -> (continuation of Deferred@112344303 after retry RPC after error@534215557) -> (continuation of Deferred@1213026672 after retry RPC after error@249639077) -> (continuation of Deferred@430125266 after retry RPC after error@1032616752) -> (continuation of Deferred@535329568 after retry RPC after error@403375200), errback=(continuation of Deferred@1534592317 after retry RPC after error@1909375867) -> (continuation of Deferred@739601181 after retry RPC after error@501257111) -> (continuation of Deferred@111815931 after retry RPC after error@33088078) -> (continuation of Deferred@112344303 after retry RPC after error@534215557) -> (continuation of Deferred@1213026672 after retry RPC after error@249639077) -> (continuation of Deferred@430125266 after retry RPC after error@1032616752) -> (continuation of Deferred@535329568 after retry RPC after error@403375200)))
03:00:08.259 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.260 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.260 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.260 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.260 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.260 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.260 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.261 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.279 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.279 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.280 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.280 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.280 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.280 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.280 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.280 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.301 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.301 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.301 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.301 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.301 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.301 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.302 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.320 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.321 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.321 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.321 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.321 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.321 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.340 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.340 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.340 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.340 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.341 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.341 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.359 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.360 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.360 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.360 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.360 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.360 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.360 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.399 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.400 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.400 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.400 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.400 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.400 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.400 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.520 [DEBUG - pool-40-thread-1] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1682889276(state=PENDING, result=null, callback=(continuation of Deferred@320730124 after retry RPC after error@109430308) -> (continuation of Deferred@1404794372 after retry RPC after error@1094113653) -> (continuation of Deferred@1039661259 after retry RPC after error@367766967) -> (continuation of Deferred@493412952 after retry RPC after error@1784835569) -> (continuation of Deferred@1758585302 after retry RPC after error@547323758) -> (continuation of Deferred@2059350791 after retry RPC after error@55297092) -> (continuation of Deferred@1685277354 after retry RPC after error@1061961029), errback=(continuation of Deferred@320730124 after retry RPC after error@109430308) -> (continuation of Deferred@1404794372 after retry RPC after error@1094113653) -> (continuation of Deferred@1039661259 after retry RPC after error@367766967) -> (continuation of Deferred@493412952 after retry RPC after error@1784835569) -> (continuation of Deferred@1758585302 after retry RPC after error@547323758) -> (continuation of Deferred@2059350791 after retry RPC after error@55297092) -> (continuation of Deferred@1685277354 after retry RPC after error@1061961029)))
03:00:08.520 [WARN - pool-40-thread-1] (HashedWheelTimer.java:729) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
03:00:08.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.541 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.541 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.541 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.541 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.541 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.541 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.542 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.561 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.561 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.561 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.561 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.561 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.561 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.562 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.580 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.581 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.581 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.581 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.581 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.581 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.600 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.600 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.600 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.600 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.600 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.600 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.619 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.620 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.620 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.620 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.620 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.620 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.620 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.659 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.660 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.660 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.660 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.660 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.660 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.660 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.700 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.700 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.700 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.700 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.700 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.701 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.719 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.720 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.720 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.720 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.720 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.720 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.720 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.721 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.721 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1482078771(state=PENDING, result=null, callback=(continuation of Deferred@2104023030 after retry RPC after error@326853242) -> (continuation of Deferred@2058381791 after retry RPC after error@400216984) -> (continuation of Deferred@750150130 after retry RPC after error@1679208108) -> (continuation of Deferred@496094560 after retry RPC after error@1544985433) -> (continuation of Deferred@1135659224 after retry RPC after error@302153164) -> (continuation of Deferred@23808033 after retry RPC after error@1775224582) -> (continuation of Deferred@746333423 after retry RPC after error@764526769), errback=(continuation of Deferred@2104023030 after retry RPC after error@326853242) -> (continuation of Deferred@2058381791 after retry RPC after error@400216984) -> (continuation of Deferred@750150130 after retry RPC after error@1679208108) -> (continuation of Deferred@496094560 after retry RPC after error@1544985433) -> (continuation of Deferred@1135659224 after retry RPC after error@302153164) -> (continuation of Deferred@23808033 after retry RPC after error@1775224582) -> (continuation of Deferred@746333423 after retry RPC after error@764526769)))
03:00:08.721 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1482078771(state=PENDING, result=null, callback=(continuation of Deferred@2104023030 after retry RPC after error@326853242) -> (continuation of Deferred@2058381791 after retry RPC after error@400216984) -> (continuation of Deferred@750150130 after retry RPC after error@1679208108) -> (continuation of Deferred@496094560 after retry RPC after error@1544985433) -> (continuation of Deferred@1135659224 after retry RPC after error@302153164) -> (continuation of Deferred@23808033 after retry RPC after error@1775224582) -> (continuation of Deferred@746333423 after retry RPC after error@764526769), errback=(continuation of Deferred@2104023030 after retry RPC after error@326853242) -> (continuation of Deferred@2058381791 after retry RPC after error@400216984) -> (continuation of Deferred@750150130 after retry RPC after error@1679208108) -> (continuation of Deferred@496094560 after retry RPC after error@1544985433) -> (continuation of Deferred@1135659224 after retry RPC after error@302153164) -> (continuation of Deferred@23808033 after retry RPC after error@1775224582) -> (continuation of Deferred@746333423 after retry RPC after error@764526769)))
03:00:08.739 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.740 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.740 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.740 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.740 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.740 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.740 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:08.755314 21343 raft_consensus.cc:534] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 56099a6022da440a81f20e6fe8903fb0)
03:00:08.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:08.755509 21343 raft_consensus.cc:556] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
03:00:08.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:08.756309 21343 leader_election.cc:290] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
03:00:08.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 03:00:08.758694 20378 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111): Network error: Client connection negotiation failed: client connection to 127.19.203.254:46111: connect: Connection refused (error 111)
03:00:08.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 03:00:08.758877 20381 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
03:00:08.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:08.758984 20381 leader_election.cc:304] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c; no voters: 56099a6022da440a81f20e6fe8903fb0, efb4c69b20bd4f9692d496c33ddbb24e
03:00:08.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:08.759248 21343 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
03:00:08.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.761 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.761 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.761 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.761 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.761 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.761 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.780 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.781 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.781 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.781 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.781 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.781 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.800 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.800 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.800 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.801 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.801 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.801 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.820 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.820 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.820 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.820 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.820 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.821 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.861 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.861 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.861 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.861 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.861 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.861 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.862 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.919 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.920 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.920 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.920 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.920 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.921 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.921 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.960 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.961 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.961 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.961 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.961 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.961 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.962 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.962 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@405248529(state=PENDING, result=null, callback=(continuation of Deferred@25039635 after retry RPC after error@88503376) -> (continuation of Deferred@864890688 after retry RPC after error@392088626) -> (continuation of Deferred@1716101953 after retry RPC after error@1343991596) -> (continuation of Deferred@179731776 after retry RPC after error@1801985912) -> (continuation of Deferred@1083243263 after retry RPC after error@1300193038) -> (continuation of Deferred@1956276124 after retry RPC after error@1604461992) -> (continuation of Deferred@404913057 after retry RPC after error@798150788), errback=(continuation of Deferred@25039635 after retry RPC after error@88503376) -> (continuation of Deferred@864890688 after retry RPC after error@392088626) -> (continuation of Deferred@1716101953 after retry RPC after error@1343991596) -> (continuation of Deferred@179731776 after retry RPC after error@1801985912) -> (continuation of Deferred@1083243263 after retry RPC after error@1300193038) -> (continuation of Deferred@1956276124 after retry RPC after error@1604461992) -> (continuation of Deferred@404913057 after retry RPC after error@798150788)))
03:00:08.962 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@405248529(state=PENDING, result=null, callback=(continuation of Deferred@25039635 after retry RPC after error@88503376) -> (continuation of Deferred@864890688 after retry RPC after error@392088626) -> (continuation of Deferred@1716101953 after retry RPC after error@1343991596) -> (continuation of Deferred@179731776 after retry RPC after error@1801985912) -> (continuation of Deferred@1083243263 after retry RPC after error@1300193038) -> (continuation of Deferred@1956276124 after retry RPC after error@1604461992) -> (continuation of Deferred@404913057 after retry RPC after error@798150788), errback=(continuation of Deferred@25039635 after retry RPC after error@88503376) -> (continuation of Deferred@864890688 after retry RPC after error@392088626) -> (continuation of Deferred@1716101953 after retry RPC after error@1343991596) -> (continuation of Deferred@179731776 after retry RPC after error@1801985912) -> (continuation of Deferred@1083243263 after retry RPC after error@1300193038) -> (continuation of Deferred@1956276124 after retry RPC after error@1604461992) -> (continuation of Deferred@404913057 after retry RPC after error@798150788)))
03:00:08.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:08.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:08.980 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.980 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.980 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:08.981 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.981 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.981 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:08.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:08.999 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.000 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.000 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.000 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.001 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.001 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.001 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.019 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.020 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.020 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.020 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.020 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.020 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.020 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.022 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.039 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.040 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.040 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.040 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.040 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.040 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.040 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.041 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.061 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.061 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.061 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.061 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.061 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.061 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.062 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.101 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.101 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.101 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.101 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.101 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.101 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.102 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.141 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.141 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.141 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.141 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.141 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.141 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.141 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.141 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.142 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.142 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.179 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.180 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.180 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.180 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.180 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.180 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.180 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.181 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] refreshing cache from master, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [163ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@903346770(state=PENDING, result=null, callback=(continuation of Deferred@1289083913 after retry RPC after error@1941982157) -> (continuation of Deferred@1716243769 after retry RPC after error@1824829128) -> (continuation of Deferred@236131021 after retry RPC after error@2138877665) -> (continuation of Deferred@1197905458 after retry RPC after error@236835576) -> (continuation of Deferred@403582092 after retry RPC after error@35623626) -> (continuation of Deferred@1899713469 after retry RPC after error@290571632) -> (continuation of Deferred@2027090330 after retry RPC after error@103134600), errback=(continuation of Deferred@1289083913 after retry RPC after error@1941982157) -> (continuation of Deferred@1716243769 after retry RPC after error@1824829128) -> (continuation of Deferred@236131021 after retry RPC after error@2138877665) -> (continuation of Deferred@1197905458 after retry RPC after error@236835576) -> (continuation of Deferred@403582092 after retry RPC after error@35623626) -> (continuation of Deferred@1899713469 after retry RPC after error@290571632) -> (continuation of Deferred@2027090330 after retry RPC after error@103134600)))
03:00:09.182 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] refreshing cache from master, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [163ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@903346770(state=PENDING, result=null, callback=(continuation of Deferred@1289083913 after retry RPC after error@1941982157) -> (continuation of Deferred@1716243769 after retry RPC after error@1824829128) -> (continuation of Deferred@236131021 after retry RPC after error@2138877665) -> (continuation of Deferred@1197905458 after retry RPC after error@236835576) -> (continuation of Deferred@403582092 after retry RPC after error@35623626) -> (continuation of Deferred@1899713469 after retry RPC after error@290571632) -> (continuation of Deferred@2027090330 after retry RPC after error@103134600), errback=(continuation of Deferred@1289083913 after retry RPC after error@1941982157) -> (continuation of Deferred@1716243769 after retry RPC after error@1824829128) -> (continuation of Deferred@236131021 after retry RPC after error@2138877665) -> (continuation of Deferred@1197905458 after retry RPC after error@236835576) -> (continuation of Deferred@403582092 after retry RPC after error@35623626) -> (continuation of Deferred@1899713469 after retry RPC after error@290571632) -> (continuation of Deferred@2027090330 after retry RPC after error@103134600)))
03:00:09.199 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.200 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.200 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.200 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.200 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.201 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.201 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.221 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.221 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.221 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.221 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.221 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.221 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.222 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.241 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.241 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.242 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.242 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.242 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.242 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.242 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.260 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.260 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.260 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.261 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.261 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.261 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.262 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.280 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.280 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.280 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.281 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.281 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.281 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.320 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.320 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.320 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.320 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.320 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.321 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.360 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.360 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.360 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.360 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.361 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.361 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.441 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.441 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.441 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.441 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.441 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.441 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.442 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2051415112(state=PENDING, result=null, callback=(continuation of Deferred@2021488166 after retry RPC after error@115030306) -> (continuation of Deferred@249343011 after retry RPC after error@1152533996) -> (continuation of Deferred@1475200794 after retry RPC after error@136373996) -> (continuation of Deferred@1677127295 after retry RPC after error@901601992) -> (continuation of Deferred@215060082 after retry RPC after error@63512194) -> (continuation of Deferred@1469615506 after retry RPC after error@145548979) -> (continuation of Deferred@143543242 after retry RPC after error@1140357463), errback=(continuation of Deferred@2021488166 after retry RPC after error@115030306) -> (continuation of Deferred@249343011 after retry RPC after error@1152533996) -> (continuation of Deferred@1475200794 after retry RPC after error@136373996) -> (continuation of Deferred@1677127295 after retry RPC after error@901601992) -> (continuation of Deferred@215060082 after retry RPC after error@63512194) -> (continuation of Deferred@1469615506 after retry RPC after error@145548979) -> (continuation of Deferred@143543242 after retry RPC after error@1140357463)))
03:00:09.442 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2051415112(state=PENDING, result=null, callback=(continuation of Deferred@2021488166 after retry RPC after error@115030306) -> (continuation of Deferred@249343011 after retry RPC after error@1152533996) -> (continuation of Deferred@1475200794 after retry RPC after error@136373996) -> (continuation of Deferred@1677127295 after retry RPC after error@901601992) -> (continuation of Deferred@215060082 after retry RPC after error@63512194) -> (continuation of Deferred@1469615506 after retry RPC after error@145548979) -> (continuation of Deferred@143543242 after retry RPC after error@1140357463), errback=(continuation of Deferred@2021488166 after retry RPC after error@115030306) -> (continuation of Deferred@249343011 after retry RPC after error@1152533996) -> (continuation of Deferred@1475200794 after retry RPC after error@136373996) -> (continuation of Deferred@1677127295 after retry RPC after error@901601992) -> (continuation of Deferred@215060082 after retry RPC after error@63512194) -> (continuation of Deferred@1469615506 after retry RPC after error@145548979) -> (continuation of Deferred@143543242 after retry RPC after error@1140357463)))
03:00:09.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.460 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.460 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.461 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.461 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.461 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.461 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.461 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.480 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.480 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.480 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.481 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.481 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.481 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.481 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.499 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.500 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.500 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.500 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.500 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.500 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.500 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.519 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.519 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.520 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.520 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.520 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.520 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.520 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.520 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.541 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.541 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.541 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.541 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.541 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.541 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.581 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.581 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.581 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.581 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.581 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.581 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.581 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.582 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.601 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.601 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.601 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.601 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.601 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.601 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.602 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.641 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.641 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.641 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.641 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.641 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.642 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.642 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.643 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.643 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1762078930(state=PENDING, result=null, callback=(continuation of Deferred@489804454 after retry RPC after error@1163942228) -> (continuation of Deferred@181254425 after retry RPC after error@1520212274) -> (continuation of Deferred@1235993637 after retry RPC after error@247897789) -> (continuation of Deferred@1772419853 after retry RPC after error@1473231362) -> (continuation of Deferred@1079166514 after retry RPC after error@2061212659) -> (continuation of Deferred@879328054 after retry RPC after error@517789715) -> (continuation of Deferred@1223290540 after retry RPC after error@1850466369), errback=(continuation of Deferred@489804454 after retry RPC after error@1163942228) -> (continuation of Deferred@181254425 after retry RPC after error@1520212274) -> (continuation of Deferred@1235993637 after retry RPC after error@247897789) -> (continuation of Deferred@1772419853 after retry RPC after error@1473231362) -> (continuation of Deferred@1079166514 after retry RPC after error@2061212659) -> (continuation of Deferred@879328054 after retry RPC after error@517789715) -> (continuation of Deferred@1223290540 after retry RPC after error@1850466369)))
03:00:09.644 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1762078930(state=PENDING, result=null, callback=(continuation of Deferred@489804454 after retry RPC after error@1163942228) -> (continuation of Deferred@181254425 after retry RPC after error@1520212274) -> (continuation of Deferred@1235993637 after retry RPC after error@247897789) -> (continuation of Deferred@1772419853 after retry RPC after error@1473231362) -> (continuation of Deferred@1079166514 after retry RPC after error@2061212659) -> (continuation of Deferred@879328054 after retry RPC after error@517789715) -> (continuation of Deferred@1223290540 after retry RPC after error@1850466369), errback=(continuation of Deferred@489804454 after retry RPC after error@1163942228) -> (continuation of Deferred@181254425 after retry RPC after error@1520212274) -> (continuation of Deferred@1235993637 after retry RPC after error@247897789) -> (continuation of Deferred@1772419853 after retry RPC after error@1473231362) -> (continuation of Deferred@1079166514 after retry RPC after error@2061212659) -> (continuation of Deferred@879328054 after retry RPC after error@517789715) -> (continuation of Deferred@1223290540 after retry RPC after error@1850466369)))
03:00:09.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.661 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.661 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.661 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.662 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.662 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.662 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.662 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.681 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.681 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.681 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.681 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.681 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.681 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.699 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.700 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.700 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.700 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.700 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.700 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.700 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.721 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.721 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.721 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.721 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.721 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.721 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.722 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.741 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.741 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.741 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.741 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.741 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.741 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.781 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.781 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.781 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.781 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.781 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.781 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.781 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.782 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.839 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.840 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.840 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.840 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.840 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.840 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.840 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.879 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.880 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.880 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.880 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.880 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.880 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.880 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.882 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1786870229(state=PENDING, result=null, callback=(continuation of Deferred@89046123 after retry RPC after error@901609677) -> (continuation of Deferred@1897999775 after retry RPC after error@269462975) -> (continuation of Deferred@1616049713 after retry RPC after error@1048023526) -> (continuation of Deferred@1187122786 after retry RPC after error@726475088) -> (continuation of Deferred@1549384054 after retry RPC after error@1579416506) -> (continuation of Deferred@138698498 after retry RPC after error@1245817083) -> (continuation of Deferred@1539451596 after retry RPC after error@444458021), errback=(continuation of Deferred@89046123 after retry RPC after error@901609677) -> (continuation of Deferred@1897999775 after retry RPC after error@269462975) -> (continuation of Deferred@1616049713 after retry RPC after error@1048023526) -> (continuation of Deferred@1187122786 after retry RPC after error@726475088) -> (continuation of Deferred@1549384054 after retry RPC after error@1579416506) -> (continuation of Deferred@138698498 after retry RPC after error@1245817083) -> (continuation of Deferred@1539451596 after retry RPC after error@444458021)))
03:00:09.882 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1786870229(state=PENDING, result=null, callback=(continuation of Deferred@89046123 after retry RPC after error@901609677) -> (continuation of Deferred@1897999775 after retry RPC after error@269462975) -> (continuation of Deferred@1616049713 after retry RPC after error@1048023526) -> (continuation of Deferred@1187122786 after retry RPC after error@726475088) -> (continuation of Deferred@1549384054 after retry RPC after error@1579416506) -> (continuation of Deferred@138698498 after retry RPC after error@1245817083) -> (continuation of Deferred@1539451596 after retry RPC after error@444458021), errback=(continuation of Deferred@89046123 after retry RPC after error@901609677) -> (continuation of Deferred@1897999775 after retry RPC after error@269462975) -> (continuation of Deferred@1616049713 after retry RPC after error@1048023526) -> (continuation of Deferred@1187122786 after retry RPC after error@726475088) -> (continuation of Deferred@1549384054 after retry RPC after error@1579416506) -> (continuation of Deferred@138698498 after retry RPC after error@1245817083) -> (continuation of Deferred@1539451596 after retry RPC after error@444458021)))
03:00:09.899 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.900 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.900 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.900 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.900 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.900 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.900 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.919 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.920 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.920 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.920 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.920 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.920 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.920 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.939 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.939 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.940 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.940 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.940 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.940 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.940 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.940 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.961 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.961 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.961 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.961 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.961 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.961 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:09.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:09.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:09.980 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.980 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.981 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.981 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.981 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:09.981 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:09.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.020 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.020 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.021 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.021 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.021 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.021 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.081 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.081 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.081 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.081 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.081 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.081 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.082 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.082 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@72624967(state=PENDING, result=null, callback=(continuation of Deferred@480136507 after retry RPC after error@1040090150) -> (continuation of Deferred@1732246632 after retry RPC after error@1108667867) -> (continuation of Deferred@739084087 after retry RPC after error@1704720676) -> (continuation of Deferred@2111800643 after retry RPC after error@406897942) -> (continuation of Deferred@1488248086 after retry RPC after error@1658609149) -> (continuation of Deferred@1671969144 after retry RPC after error@2103326830), errback=(continuation of Deferred@480136507 after retry RPC after error@1040090150) -> (continuation of Deferred@1732246632 after retry RPC after error@1108667867) -> (continuation of Deferred@739084087 after retry RPC after error@1704720676) -> (continuation of Deferred@2111800643 after retry RPC after error@406897942) -> (continuation of Deferred@1488248086 after retry RPC after error@1658609149) -> (continuation of Deferred@1671969144 after retry RPC after error@2103326830)))
03:00:10.082 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@72624967(state=PENDING, result=null, callback=(continuation of Deferred@480136507 after retry RPC after error@1040090150) -> (continuation of Deferred@1732246632 after retry RPC after error@1108667867) -> (continuation of Deferred@739084087 after retry RPC after error@1704720676) -> (continuation of Deferred@2111800643 after retry RPC after error@406897942) -> (continuation of Deferred@1488248086 after retry RPC after error@1658609149) -> (continuation of Deferred@1671969144 after retry RPC after error@2103326830), errback=(continuation of Deferred@480136507 after retry RPC after error@1040090150) -> (continuation of Deferred@1732246632 after retry RPC after error@1108667867) -> (continuation of Deferred@739084087 after retry RPC after error@1704720676) -> (continuation of Deferred@2111800643 after retry RPC after error@406897942) -> (continuation of Deferred@1488248086 after retry RPC after error@1658609149) -> (continuation of Deferred@1671969144 after retry RPC after error@2103326830)))
03:00:10.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.101 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.101 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.101 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.101 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.101 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.101 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.102 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.121 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.121 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.121 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.121 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.121 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.121 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.122 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.141 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.141 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.141 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.141 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.141 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.141 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.141 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.160 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.160 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.160 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.160 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.161 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.161 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.180 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.180 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.180 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.180 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.180 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.180 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.199 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.200 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.200 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.200 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.200 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.200 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.200 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.280 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.280 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.281 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.281 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.281 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.281 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.282 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.300 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.300 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.301 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.301 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.301 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.301 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.302 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [199ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1312158093(state=PENDING, result=null, callback=(continuation of Deferred@830597914 after retry RPC after error@1619866185) -> (continuation of Deferred@865767765 after retry RPC after error@1388492594) -> (continuation of Deferred@1814384459 after retry RPC after error@1706244092) -> (continuation of Deferred@38015526 after retry RPC after error@1173905019) -> (continuation of Deferred@56696472 after retry RPC after error@2140263903) -> (continuation of Deferred@1575365929 after retry RPC after error@1516944232) -> (continuation of Deferred@1366621027 after retry RPC after error@549123007), errback=(continuation of Deferred@830597914 after retry RPC after error@1619866185) -> (continuation of Deferred@865767765 after retry RPC after error@1388492594) -> (continuation of Deferred@1814384459 after retry RPC after error@1706244092) -> (continuation of Deferred@38015526 after retry RPC after error@1173905019) -> (continuation of Deferred@56696472 after retry RPC after error@2140263903) -> (continuation of Deferred@1575365929 after retry RPC after error@1516944232) -> (continuation of Deferred@1366621027 after retry RPC after error@549123007)))
03:00:10.302 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [199ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1312158093(state=PENDING, result=null, callback=(continuation of Deferred@830597914 after retry RPC after error@1619866185) -> (continuation of Deferred@865767765 after retry RPC after error@1388492594) -> (continuation of Deferred@1814384459 after retry RPC after error@1706244092) -> (continuation of Deferred@38015526 after retry RPC after error@1173905019) -> (continuation of Deferred@56696472 after retry RPC after error@2140263903) -> (continuation of Deferred@1575365929 after retry RPC after error@1516944232) -> (continuation of Deferred@1366621027 after retry RPC after error@549123007), errback=(continuation of Deferred@830597914 after retry RPC after error@1619866185) -> (continuation of Deferred@865767765 after retry RPC after error@1388492594) -> (continuation of Deferred@1814384459 after retry RPC after error@1706244092) -> (continuation of Deferred@38015526 after retry RPC after error@1173905019) -> (continuation of Deferred@56696472 after retry RPC after error@2140263903) -> (continuation of Deferred@1575365929 after retry RPC after error@1516944232) -> (continuation of Deferred@1366621027 after retry RPC after error@549123007)))
03:00:10.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.320 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.320 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.320 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.320 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.320 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.320 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.339 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.340 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.340 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.340 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.340 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.340 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.340 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.361 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.361 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.361 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.361 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.361 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.361 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.362 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.381 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.381 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.381 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.381 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.381 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.381 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.382 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.401 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.401 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.401 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.401 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.401 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.401 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.402 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.441 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.441 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.441 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.441 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.441 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.441 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.499 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.500 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.500 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.500 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.500 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.500 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.500 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.501 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1180443543(state=PENDING, result=null, callback=(continuation of Deferred@1068832158 after retry RPC after error@2057205362) -> (continuation of Deferred@1237532442 after retry RPC after error@1847917870) -> (continuation of Deferred@2032798865 after retry RPC after error@1880016928) -> (continuation of Deferred@551883476 after retry RPC after error@1767982699) -> (continuation of Deferred@1074172242 after retry RPC after error@1179586234) -> (continuation of Deferred@1320474389 after retry RPC after error@755339958), errback=(continuation of Deferred@1068832158 after retry RPC after error@2057205362) -> (continuation of Deferred@1237532442 after retry RPC after error@1847917870) -> (continuation of Deferred@2032798865 after retry RPC after error@1880016928) -> (continuation of Deferred@551883476 after retry RPC after error@1767982699) -> (continuation of Deferred@1074172242 after retry RPC after error@1179586234) -> (continuation of Deferred@1320474389 after retry RPC after error@755339958)))
03:00:10.502 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1180443543(state=PENDING, result=null, callback=(continuation of Deferred@1068832158 after retry RPC after error@2057205362) -> (continuation of Deferred@1237532442 after retry RPC after error@1847917870) -> (continuation of Deferred@2032798865 after retry RPC after error@1880016928) -> (continuation of Deferred@551883476 after retry RPC after error@1767982699) -> (continuation of Deferred@1074172242 after retry RPC after error@1179586234) -> (continuation of Deferred@1320474389 after retry RPC after error@755339958), errback=(continuation of Deferred@1068832158 after retry RPC after error@2057205362) -> (continuation of Deferred@1237532442 after retry RPC after error@1847917870) -> (continuation of Deferred@2032798865 after retry RPC after error@1880016928) -> (continuation of Deferred@551883476 after retry RPC after error@1767982699) -> (continuation of Deferred@1074172242 after retry RPC after error@1179586234) -> (continuation of Deferred@1320474389 after retry RPC after error@755339958)))
03:00:10.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.521 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.521 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.521 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.521 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.521 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.521 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.522 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.541 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.541 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.541 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.541 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.541 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.541 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.542 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.560 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.560 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.560 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.561 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.561 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.561 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.561 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.580 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.580 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.580 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.580 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.581 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.581 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.582 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.599 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.600 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.600 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.600 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.600 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.600 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.600 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.639 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.640 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.640 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.640 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.640 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.640 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.640 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.659 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.660 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.660 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.660 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.660 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.660 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.660 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.661 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1066542410(state=PENDING, result=null, callback=(continuation of Deferred@392914467 after retry RPC after error@1958838023) -> (continuation of Deferred@468581162 after retry RPC after error@905655656) -> (continuation of Deferred@2045577098 after retry RPC after error@103507703) -> (continuation of Deferred@1919678899 after retry RPC after error@509072099) -> (continuation of Deferred@253188084 after retry RPC after error@1381401663) -> (continuation of Deferred@2145212839 after retry RPC after error@1606241397), errback=(continuation of Deferred@392914467 after retry RPC after error@1958838023) -> (continuation of Deferred@468581162 after retry RPC after error@905655656) -> (continuation of Deferred@2045577098 after retry RPC after error@103507703) -> (continuation of Deferred@1919678899 after retry RPC after error@509072099) -> (continuation of Deferred@253188084 after retry RPC after error@1381401663) -> (continuation of Deferred@2145212839 after retry RPC after error@1606241397)))
03:00:10.662 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1066542410(state=PENDING, result=null, callback=(continuation of Deferred@392914467 after retry RPC after error@1958838023) -> (continuation of Deferred@468581162 after retry RPC after error@905655656) -> (continuation of Deferred@2045577098 after retry RPC after error@103507703) -> (continuation of Deferred@1919678899 after retry RPC after error@509072099) -> (continuation of Deferred@253188084 after retry RPC after error@1381401663) -> (continuation of Deferred@2145212839 after retry RPC after error@1606241397), errback=(continuation of Deferred@392914467 after retry RPC after error@1958838023) -> (continuation of Deferred@468581162 after retry RPC after error@905655656) -> (continuation of Deferred@2045577098 after retry RPC after error@103507703) -> (continuation of Deferred@1919678899 after retry RPC after error@509072099) -> (continuation of Deferred@253188084 after retry RPC after error@1381401663) -> (continuation of Deferred@2145212839 after retry RPC after error@1606241397)))
03:00:10.679 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.680 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.680 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.680 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.680 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.680 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.680 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.701 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.701 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.701 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.701 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.701 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.701 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.701 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.702 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.719 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.720 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.720 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.720 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.720 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.720 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.720 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.721 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.739 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.740 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.740 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.740 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.740 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.740 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.740 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.759 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.760 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.760 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.760 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.760 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.760 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.760 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.799 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.800 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.800 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.800 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.800 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.800 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.800 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.839 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.840 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.840 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.840 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.840 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.840 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.840 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.840 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.840 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.940 [DEBUG - pool-40-thread-1] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@421040015(state=PENDING, result=null, callback=(continuation of Deferred@1157111142 after retry RPC after error@1883486681) -> (continuation of Deferred@125118229 after retry RPC after error@1523364995) -> (continuation of Deferred@1591447953 after retry RPC after error@867923682) -> (continuation of Deferred@492352015 after retry RPC after error@579822987) -> (continuation of Deferred@551169874 after retry RPC after error@1212039629) -> (continuation of Deferred@279861775 after retry RPC after error@1230296748) -> (continuation of Deferred@939905131 after retry RPC after error@34590351), errback=(continuation of Deferred@1157111142 after retry RPC after error@1883486681) -> (continuation of Deferred@125118229 after retry RPC after error@1523364995) -> (continuation of Deferred@1591447953 after retry RPC after error@867923682) -> (continuation of Deferred@492352015 after retry RPC after error@579822987) -> (continuation of Deferred@551169874 after retry RPC after error@1212039629) -> (continuation of Deferred@279861775 after retry RPC after error@1230296748) -> (continuation of Deferred@939905131 after retry RPC after error@34590351)))
03:00:10.941 [WARN - pool-40-thread-1] (HashedWheelTimer.java:729) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
03:00:10.959 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.960 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.960 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.960 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.960 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.960 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.960 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:10.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:10.981 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:10.981 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.981 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.981 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:10.981 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.981 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.981 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:10.982 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.001 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.001 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.001 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.001 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.001 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.001 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.001 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.002 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.021 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.021 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.021 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.021 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.021 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.021 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.022 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.041 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.041 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.041 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.041 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.041 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.041 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.041 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.060 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.060 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.060 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.060 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.061 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.061 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.100 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.100 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.100 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.101 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.101 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.101 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.140 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.140 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.140 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.140 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.140 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.140 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.141 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.141 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2020102611(state=PENDING, result=null, callback=(continuation of Deferred@786068848 after retry RPC after error@1710484526) -> (continuation of Deferred@629916932 after retry RPC after error@534245039) -> (continuation of Deferred@1658550736 after retry RPC after error@406022969) -> (continuation of Deferred@1676361438 after retry RPC after error@1130960538) -> (continuation of Deferred@2051400414 after retry RPC after error@2032940044) -> (continuation of Deferred@1133246521 after retry RPC after error@1222503188) -> (continuation of Deferred@492549059 after retry RPC after error@89090096), errback=(continuation of Deferred@786068848 after retry RPC after error@1710484526) -> (continuation of Deferred@629916932 after retry RPC after error@534245039) -> (continuation of Deferred@1658550736 after retry RPC after error@406022969) -> (continuation of Deferred@1676361438 after retry RPC after error@1130960538) -> (continuation of Deferred@2051400414 after retry RPC after error@2032940044) -> (continuation of Deferred@1133246521 after retry RPC after error@1222503188) -> (continuation of Deferred@492549059 after retry RPC after error@89090096)))
03:00:11.142 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2020102611(state=PENDING, result=null, callback=(continuation of Deferred@786068848 after retry RPC after error@1710484526) -> (continuation of Deferred@629916932 after retry RPC after error@534245039) -> (continuation of Deferred@1658550736 after retry RPC after error@406022969) -> (continuation of Deferred@1676361438 after retry RPC after error@1130960538) -> (continuation of Deferred@2051400414 after retry RPC after error@2032940044) -> (continuation of Deferred@1133246521 after retry RPC after error@1222503188) -> (continuation of Deferred@492549059 after retry RPC after error@89090096), errback=(continuation of Deferred@786068848 after retry RPC after error@1710484526) -> (continuation of Deferred@629916932 after retry RPC after error@534245039) -> (continuation of Deferred@1658550736 after retry RPC after error@406022969) -> (continuation of Deferred@1676361438 after retry RPC after error@1130960538) -> (continuation of Deferred@2051400414 after retry RPC after error@2032940044) -> (continuation of Deferred@1133246521 after retry RPC after error@1222503188) -> (continuation of Deferred@492549059 after retry RPC after error@89090096)))
03:00:11.159 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.160 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.160 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.160 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.161 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.161 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.161 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.179 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.180 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.180 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.180 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.180 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.180 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.180 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.199 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.200 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.200 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.200 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.200 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.200 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.200 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.219 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.220 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.220 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.220 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.220 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.220 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.220 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.241 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.241 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.241 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.241 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.241 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.241 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.242 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.261 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.261 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.261 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.261 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.261 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.261 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.261 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.301 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.301 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.301 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.301 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.301 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.301 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.320 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.320 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.320 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.320 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.320 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.321 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.360 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.360 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.360 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.360 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.361 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.361 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.362 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@253782137(state=PENDING, result=null, callback=(continuation of Deferred@262096682 after retry RPC after error@968077785) -> (continuation of Deferred@1232750190 after retry RPC after error@1167428760) -> (continuation of Deferred@809903383 after retry RPC after error@350189503) -> (continuation of Deferred@252512892 after retry RPC after error@1721083465) -> (continuation of Deferred@1736067345 after retry RPC after error@150448334) -> (continuation of Deferred@1867386182 after retry RPC after error@614615304) -> (continuation of Deferred@287686220 after retry RPC after error@614581630) -> (continuation of Deferred@999999633 after retry RPC after error@222706915), errback=(continuation of Deferred@262096682 after retry RPC after error@968077785) -> (continuation of Deferred@1232750190 after retry RPC after error@1167428760) -> (continuation of Deferred@809903383 after retry RPC after error@350189503) -> (continuation of Deferred@252512892 after retry RPC after error@1721083465) -> (continuation of Deferred@1736067345 after retry RPC after error@150448334) -> (continuation of Deferred@1867386182 after retry RPC after error@614615304) -> (continuation of Deferred@287686220 after retry RPC after error@614581630) -> (continuation of Deferred@999999633 after retry RPC after error@222706915)))
03:00:11.362 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@253782137(state=PENDING, result=null, callback=(continuation of Deferred@262096682 after retry RPC after error@968077785) -> (continuation of Deferred@1232750190 after retry RPC after error@1167428760) -> (continuation of Deferred@809903383 after retry RPC after error@350189503) -> (continuation of Deferred@252512892 after retry RPC after error@1721083465) -> (continuation of Deferred@1736067345 after retry RPC after error@150448334) -> (continuation of Deferred@1867386182 after retry RPC after error@614615304) -> (continuation of Deferred@287686220 after retry RPC after error@614581630) -> (continuation of Deferred@999999633 after retry RPC after error@222706915), errback=(continuation of Deferred@262096682 after retry RPC after error@968077785) -> (continuation of Deferred@1232750190 after retry RPC after error@1167428760) -> (continuation of Deferred@809903383 after retry RPC after error@350189503) -> (continuation of Deferred@252512892 after retry RPC after error@1721083465) -> (continuation of Deferred@1736067345 after retry RPC after error@150448334) -> (continuation of Deferred@1867386182 after retry RPC after error@614615304) -> (continuation of Deferred@287686220 after retry RPC after error@614581630) -> (continuation of Deferred@999999633 after retry RPC after error@222706915)))
03:00:11.379 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.380 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.380 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.380 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.380 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.380 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.380 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.399 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.400 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.400 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.400 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.400 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.400 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.400 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.419 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.420 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.420 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.420 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.420 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.420 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.420 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.421 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.440 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.440 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.440 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.440 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.440 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.440 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.459 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.460 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.460 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.460 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.461 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.461 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.461 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.461 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.480 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.480 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.480 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.480 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.481 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.481 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.482 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.541 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.541 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.541 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.541 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.541 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.541 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.542 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1992167431(state=PENDING, result=null, callback=(continuation of Deferred@1252231959 after retry RPC after error@317744502) -> (continuation of Deferred@1709679290 after retry RPC after error@1675601961) -> (continuation of Deferred@896695414 after retry RPC after error@607675515) -> (continuation of Deferred@2081611828 after retry RPC after error@1653768575) -> (continuation of Deferred@1143324823 after retry RPC after error@601456596) -> (continuation of Deferred@1218521307 after retry RPC after error@1045348809), errback=(continuation of Deferred@1252231959 after retry RPC after error@317744502) -> (continuation of Deferred@1709679290 after retry RPC after error@1675601961) -> (continuation of Deferred@896695414 after retry RPC after error@607675515) -> (continuation of Deferred@2081611828 after retry RPC after error@1653768575) -> (continuation of Deferred@1143324823 after retry RPC after error@601456596) -> (continuation of Deferred@1218521307 after retry RPC after error@1045348809)))
03:00:11.542 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1992167431(state=PENDING, result=null, callback=(continuation of Deferred@1252231959 after retry RPC after error@317744502) -> (continuation of Deferred@1709679290 after retry RPC after error@1675601961) -> (continuation of Deferred@896695414 after retry RPC after error@607675515) -> (continuation of Deferred@2081611828 after retry RPC after error@1653768575) -> (continuation of Deferred@1143324823 after retry RPC after error@601456596) -> (continuation of Deferred@1218521307 after retry RPC after error@1045348809), errback=(continuation of Deferred@1252231959 after retry RPC after error@317744502) -> (continuation of Deferred@1709679290 after retry RPC after error@1675601961) -> (continuation of Deferred@896695414 after retry RPC after error@607675515) -> (continuation of Deferred@2081611828 after retry RPC after error@1653768575) -> (continuation of Deferred@1143324823 after retry RPC after error@601456596) -> (continuation of Deferred@1218521307 after retry RPC after error@1045348809)))
03:00:11.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.560 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.560 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.560 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.561 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.561 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.561 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.561 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.580 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.580 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.580 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.580 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.580 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.580 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.599 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.600 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.600 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.600 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.600 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.600 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.600 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.619 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.620 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.620 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.620 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.620 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.620 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.621 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.639 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.640 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.640 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.640 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.640 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.640 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.640 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.679 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.680 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.680 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.680 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.680 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.680 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.680 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.739 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.740 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.740 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.740 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.740 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.740 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.740 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.741 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1298542751(state=PENDING, result=null, callback=(continuation of Deferred@581457306 after retry RPC after error@1010048708) -> (continuation of Deferred@748242842 after retry RPC after error@1351055454) -> (continuation of Deferred@811856721 after retry RPC after error@750590375) -> (continuation of Deferred@1249787453 after retry RPC after error@558951076) -> (continuation of Deferred@235306148 after retry RPC after error@1244802274) -> (continuation of Deferred@1904639055 after retry RPC after error@1470479633), errback=(continuation of Deferred@581457306 after retry RPC after error@1010048708) -> (continuation of Deferred@748242842 after retry RPC after error@1351055454) -> (continuation of Deferred@811856721 after retry RPC after error@750590375) -> (continuation of Deferred@1249787453 after retry RPC after error@558951076) -> (continuation of Deferred@235306148 after retry RPC after error@1244802274) -> (continuation of Deferred@1904639055 after retry RPC after error@1470479633)))
03:00:11.742 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1298542751(state=PENDING, result=null, callback=(continuation of Deferred@581457306 after retry RPC after error@1010048708) -> (continuation of Deferred@748242842 after retry RPC after error@1351055454) -> (continuation of Deferred@811856721 after retry RPC after error@750590375) -> (continuation of Deferred@1249787453 after retry RPC after error@558951076) -> (continuation of Deferred@235306148 after retry RPC after error@1244802274) -> (continuation of Deferred@1904639055 after retry RPC after error@1470479633), errback=(continuation of Deferred@581457306 after retry RPC after error@1010048708) -> (continuation of Deferred@748242842 after retry RPC after error@1351055454) -> (continuation of Deferred@811856721 after retry RPC after error@750590375) -> (continuation of Deferred@1249787453 after retry RPC after error@558951076) -> (continuation of Deferred@235306148 after retry RPC after error@1244802274) -> (continuation of Deferred@1904639055 after retry RPC after error@1470479633)))
03:00:11.759 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.760 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.760 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.760 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.760 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.760 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.760 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.781 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.781 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.781 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.781 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.781 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.781 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.781 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.782 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.799 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.799 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.800 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.800 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.800 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.800 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.800 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.800 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.821 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.821 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.821 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.821 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.821 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.821 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.822 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.839 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.839 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.840 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.840 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.840 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.840 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.840 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.840 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.881 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.881 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.881 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.881 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.881 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.881 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.881 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.882 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.960 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.960 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.961 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.961 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.961 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.961 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.962 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@568732811(state=PENDING, result=null, callback=(continuation of Deferred@1250557596 after retry RPC after error@1612886574) -> (continuation of Deferred@917173962 after retry RPC after error@641210080) -> (continuation of Deferred@591421861 after retry RPC after error@1443234849) -> (continuation of Deferred@939242637 after retry RPC after error@1360149380) -> (continuation of Deferred@1912546634 after retry RPC after error@398327896) -> (continuation of Deferred@2106294782 after retry RPC after error@1023761360), errback=(continuation of Deferred@1250557596 after retry RPC after error@1612886574) -> (continuation of Deferred@917173962 after retry RPC after error@641210080) -> (continuation of Deferred@591421861 after retry RPC after error@1443234849) -> (continuation of Deferred@939242637 after retry RPC after error@1360149380) -> (continuation of Deferred@1912546634 after retry RPC after error@398327896) -> (continuation of Deferred@2106294782 after retry RPC after error@1023761360)))
03:00:11.962 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@568732811(state=PENDING, result=null, callback=(continuation of Deferred@1250557596 after retry RPC after error@1612886574) -> (continuation of Deferred@917173962 after retry RPC after error@641210080) -> (continuation of Deferred@591421861 after retry RPC after error@1443234849) -> (continuation of Deferred@939242637 after retry RPC after error@1360149380) -> (continuation of Deferred@1912546634 after retry RPC after error@398327896) -> (continuation of Deferred@2106294782 after retry RPC after error@1023761360), errback=(continuation of Deferred@1250557596 after retry RPC after error@1612886574) -> (continuation of Deferred@917173962 after retry RPC after error@641210080) -> (continuation of Deferred@591421861 after retry RPC after error@1443234849) -> (continuation of Deferred@939242637 after retry RPC after error@1360149380) -> (continuation of Deferred@1912546634 after retry RPC after error@398327896) -> (continuation of Deferred@2106294782 after retry RPC after error@1023761360)))
03:00:11.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:11.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:11.980 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.980 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.980 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:11.981 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.981 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.981 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:11.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:11.999 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.000 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.000 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.000 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.000 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.000 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.000 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.021 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.021 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.021 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.021 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.021 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.021 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.022 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.041 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.041 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.041 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.041 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.041 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.041 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.042 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.060 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.061 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.061 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.061 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.061 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.061 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.101 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.101 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.101 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.101 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.101 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.101 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.121 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.121 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.121 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.121 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.121 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.121 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.122 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.122 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@379737482(state=PENDING, result=null, callback=(continuation of Deferred@1482461524 after retry RPC after error@418873373) -> (continuation of Deferred@798281928 after retry RPC after error@731831989) -> (continuation of Deferred@401882275 after retry RPC after error@866627880) -> (continuation of Deferred@977024260 after retry RPC after error@1737113472) -> (continuation of Deferred@1370294006 after retry RPC after error@1603130138) -> (continuation of Deferred@71457274 after retry RPC after error@838996173), errback=(continuation of Deferred@1482461524 after retry RPC after error@418873373) -> (continuation of Deferred@798281928 after retry RPC after error@731831989) -> (continuation of Deferred@401882275 after retry RPC after error@866627880) -> (continuation of Deferred@977024260 after retry RPC after error@1737113472) -> (continuation of Deferred@1370294006 after retry RPC after error@1603130138) -> (continuation of Deferred@71457274 after retry RPC after error@838996173)))
03:00:12.122 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@379737482(state=PENDING, result=null, callback=(continuation of Deferred@1482461524 after retry RPC after error@418873373) -> (continuation of Deferred@798281928 after retry RPC after error@731831989) -> (continuation of Deferred@401882275 after retry RPC after error@866627880) -> (continuation of Deferred@977024260 after retry RPC after error@1737113472) -> (continuation of Deferred@1370294006 after retry RPC after error@1603130138) -> (continuation of Deferred@71457274 after retry RPC after error@838996173), errback=(continuation of Deferred@1482461524 after retry RPC after error@418873373) -> (continuation of Deferred@798281928 after retry RPC after error@731831989) -> (continuation of Deferred@401882275 after retry RPC after error@866627880) -> (continuation of Deferred@977024260 after retry RPC after error@1737113472) -> (continuation of Deferred@1370294006 after retry RPC after error@1603130138) -> (continuation of Deferred@71457274 after retry RPC after error@838996173)))
03:00:12.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.141 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.141 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.141 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.141 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.141 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.141 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.141 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.159 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.160 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.160 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.160 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.160 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.160 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.160 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.179 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.180 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.180 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.180 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.180 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.180 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.180 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.201 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.201 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.201 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.201 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.201 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.201 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.201 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.202 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.221 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.221 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.221 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.221 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.221 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.221 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.222 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.261 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.261 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.261 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.261 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.261 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.261 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.262 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.299 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.300 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.300 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.300 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.300 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.300 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.301 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.301 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@740175491(state=PENDING, result=null, callback=(continuation of Deferred@246994869 after retry RPC after error@214823933) -> (continuation of Deferred@523646953 after retry RPC after error@1115777210) -> (continuation of Deferred@172755488 after retry RPC after error@1769816719) -> (continuation of Deferred@1197127990 after retry RPC after error@240639455) -> (continuation of Deferred@1679870783 after retry RPC after error@1741329422) -> (continuation of Deferred@1967854280 after retry RPC after error@1161990299), errback=(continuation of Deferred@246994869 after retry RPC after error@214823933) -> (continuation of Deferred@523646953 after retry RPC after error@1115777210) -> (continuation of Deferred@172755488 after retry RPC after error@1769816719) -> (continuation of Deferred@1197127990 after retry RPC after error@240639455) -> (continuation of Deferred@1679870783 after retry RPC after error@1741329422) -> (continuation of Deferred@1967854280 after retry RPC after error@1161990299)))
03:00:12.302 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@740175491(state=PENDING, result=null, callback=(continuation of Deferred@246994869 after retry RPC after error@214823933) -> (continuation of Deferred@523646953 after retry RPC after error@1115777210) -> (continuation of Deferred@172755488 after retry RPC after error@1769816719) -> (continuation of Deferred@1197127990 after retry RPC after error@240639455) -> (continuation of Deferred@1679870783 after retry RPC after error@1741329422) -> (continuation of Deferred@1967854280 after retry RPC after error@1161990299), errback=(continuation of Deferred@246994869 after retry RPC after error@214823933) -> (continuation of Deferred@523646953 after retry RPC after error@1115777210) -> (continuation of Deferred@172755488 after retry RPC after error@1769816719) -> (continuation of Deferred@1197127990 after retry RPC after error@240639455) -> (continuation of Deferred@1679870783 after retry RPC after error@1741329422) -> (continuation of Deferred@1967854280 after retry RPC after error@1161990299)))
03:00:12.319 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.320 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.320 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.320 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.320 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.320 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.320 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.341 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.341 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.341 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.341 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.341 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.341 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.341 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.342 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.361 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.361 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.361 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.361 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.361 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.361 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.362 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.381 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.381 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.381 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.381 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.381 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.381 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.401 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.401 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.401 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.401 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.401 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.401 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.420 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.420 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.421 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.421 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.421 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.421 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.421 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.460 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.461 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.461 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.461 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.461 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.461 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.462 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.561 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.561 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.561 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.561 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.561 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.561 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.561 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.562 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.562 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@21406656(state=PENDING, result=null, callback=(continuation of Deferred@395401854 after retry RPC after error@687334621) -> (continuation of Deferred@2057598110 after retry RPC after error@1313378473) -> (continuation of Deferred@1402433972 after retry RPC after error@1146205719) -> (continuation of Deferred@495463047 after retry RPC after error@391068743) -> (continuation of Deferred@2134767051 after retry RPC after error@915739957) -> (continuation of Deferred@1779817842 after retry RPC after error@81164659) -> (continuation of Deferred@349521807 after retry RPC after error@175625373), errback=(continuation of Deferred@395401854 after retry RPC after error@687334621) -> (continuation of Deferred@2057598110 after retry RPC after error@1313378473) -> (continuation of Deferred@1402433972 after retry RPC after error@1146205719) -> (continuation of Deferred@495463047 after retry RPC after error@391068743) -> (continuation of Deferred@2134767051 after retry RPC after error@915739957) -> (continuation of Deferred@1779817842 after retry RPC after error@81164659) -> (continuation of Deferred@349521807 after retry RPC after error@175625373)))
03:00:12.566 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [243ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@21406656(state=PENDING, result=null, callback=(continuation of Deferred@395401854 after retry RPC after error@687334621) -> (continuation of Deferred@2057598110 after retry RPC after error@1313378473) -> (continuation of Deferred@1402433972 after retry RPC after error@1146205719) -> (continuation of Deferred@495463047 after retry RPC after error@391068743) -> (continuation of Deferred@2134767051 after retry RPC after error@915739957) -> (continuation of Deferred@1779817842 after retry RPC after error@81164659) -> (continuation of Deferred@349521807 after retry RPC after error@175625373), errback=(continuation of Deferred@395401854 after retry RPC after error@687334621) -> (continuation of Deferred@2057598110 after retry RPC after error@1313378473) -> (continuation of Deferred@1402433972 after retry RPC after error@1146205719) -> (continuation of Deferred@495463047 after retry RPC after error@391068743) -> (continuation of Deferred@2134767051 after retry RPC after error@915739957) -> (continuation of Deferred@1779817842 after retry RPC after error@81164659) -> (continuation of Deferred@349521807 after retry RPC after error@175625373)))
03:00:12.579 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.580 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.580 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.580 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.580 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.581 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.581 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.601 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.601 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.601 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.601 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.601 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.601 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.602 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.621 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.621 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.621 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.621 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.621 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.621 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.622 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.641 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.641 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.641 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.641 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.641 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.641 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.642 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.660 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.661 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.661 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.661 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.661 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.661 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.680 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.680 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.680 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.680 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.680 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.681 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.741 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.741 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.741 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.741 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.741 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.741 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.742 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1451810812(state=PENDING, result=null, callback=(continuation of Deferred@322376865 after retry RPC after error@1444115838) -> (continuation of Deferred@417435293 after retry RPC after error@143054146) -> (continuation of Deferred@599739119 after retry RPC after error@1850372963) -> (continuation of Deferred@2051771729 after retry RPC after error@1155103647) -> (continuation of Deferred@501787097 after retry RPC after error@537133164) -> (continuation of Deferred@109260856 after retry RPC after error@151631604), errback=(continuation of Deferred@322376865 after retry RPC after error@1444115838) -> (continuation of Deferred@417435293 after retry RPC after error@143054146) -> (continuation of Deferred@599739119 after retry RPC after error@1850372963) -> (continuation of Deferred@2051771729 after retry RPC after error@1155103647) -> (continuation of Deferred@501787097 after retry RPC after error@537133164) -> (continuation of Deferred@109260856 after retry RPC after error@151631604)))
03:00:12.742 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1451810812(state=PENDING, result=null, callback=(continuation of Deferred@322376865 after retry RPC after error@1444115838) -> (continuation of Deferred@417435293 after retry RPC after error@143054146) -> (continuation of Deferred@599739119 after retry RPC after error@1850372963) -> (continuation of Deferred@2051771729 after retry RPC after error@1155103647) -> (continuation of Deferred@501787097 after retry RPC after error@537133164) -> (continuation of Deferred@109260856 after retry RPC after error@151631604), errback=(continuation of Deferred@322376865 after retry RPC after error@1444115838) -> (continuation of Deferred@417435293 after retry RPC after error@143054146) -> (continuation of Deferred@599739119 after retry RPC after error@1850372963) -> (continuation of Deferred@2051771729 after retry RPC after error@1155103647) -> (continuation of Deferred@501787097 after retry RPC after error@537133164) -> (continuation of Deferred@109260856 after retry RPC after error@151631604)))
03:00:12.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.760 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.760 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.761 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.761 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.761 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.761 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.780 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.780 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.780 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.781 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.781 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.781 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.800 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.800 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.800 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.801 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.801 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.801 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.821 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.821 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.821 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.821 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.821 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.821 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.821 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.822 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.840 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.840 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.840 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.840 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.840 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.841 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.859 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.860 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.860 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.860 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.860 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.860 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.860 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.899 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.900 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.900 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.900 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.900 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.900 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.900 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.939 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.940 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.940 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.940 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.940 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.940 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.940 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.941 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1611213233(state=PENDING, result=null, callback=(continuation of Deferred@834853514 after retry RPC after error@279126310) -> (continuation of Deferred@538997968 after retry RPC after error@164464068) -> (continuation of Deferred@779884797 after retry RPC after error@668972549) -> (continuation of Deferred@83546330 after retry RPC after error@1635554852) -> (continuation of Deferred@797561094 after retry RPC after error@1975264620) -> (continuation of Deferred@655513345 after retry RPC after error@576338033) -> (continuation of Deferred@1127931101 after retry RPC after error@1463355027), errback=(continuation of Deferred@834853514 after retry RPC after error@279126310) -> (continuation of Deferred@538997968 after retry RPC after error@164464068) -> (continuation of Deferred@779884797 after retry RPC after error@668972549) -> (continuation of Deferred@83546330 after retry RPC after error@1635554852) -> (continuation of Deferred@797561094 after retry RPC after error@1975264620) -> (continuation of Deferred@655513345 after retry RPC after error@576338033) -> (continuation of Deferred@1127931101 after retry RPC after error@1463355027)))
03:00:12.941 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1611213233(state=PENDING, result=null, callback=(continuation of Deferred@834853514 after retry RPC after error@279126310) -> (continuation of Deferred@538997968 after retry RPC after error@164464068) -> (continuation of Deferred@779884797 after retry RPC after error@668972549) -> (continuation of Deferred@83546330 after retry RPC after error@1635554852) -> (continuation of Deferred@797561094 after retry RPC after error@1975264620) -> (continuation of Deferred@655513345 after retry RPC after error@576338033) -> (continuation of Deferred@1127931101 after retry RPC after error@1463355027), errback=(continuation of Deferred@834853514 after retry RPC after error@279126310) -> (continuation of Deferred@538997968 after retry RPC after error@164464068) -> (continuation of Deferred@779884797 after retry RPC after error@668972549) -> (continuation of Deferred@83546330 after retry RPC after error@1635554852) -> (continuation of Deferred@797561094 after retry RPC after error@1975264620) -> (continuation of Deferred@655513345 after retry RPC after error@576338033) -> (continuation of Deferred@1127931101 after retry RPC after error@1463355027)))
03:00:12.959 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.960 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.960 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.960 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.960 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.960 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.960 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:12.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:12.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:12.981 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.981 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.981 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.981 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:12.981 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.981 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:12.982 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.001 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.001 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.001 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.001 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.001 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.001 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.002 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.021 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.021 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.021 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.021 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.021 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.021 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.040 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.041 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.041 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.041 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.041 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.041 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.041 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.041 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.041 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.042 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.081 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.081 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.081 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.081 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.081 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.081 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.082 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.100 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.100 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.100 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.100 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.100 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.101 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.160 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.160 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.161 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.161 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.161 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.161 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.161 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@972289117(state=PENDING, result=null, callback=(continuation of Deferred@342395714 after retry RPC after error@846627864) -> (continuation of Deferred@326817581 after retry RPC after error@898467843) -> (continuation of Deferred@1701783403 after retry RPC after error@1665083802) -> (continuation of Deferred@649677167 after retry RPC after error@2133655595) -> (continuation of Deferred@1675965658 after retry RPC after error@2039263941) -> (continuation of Deferred@353533610 after retry RPC after error@44632187) -> (continuation of Deferred@1218494224 after retry RPC after error@1420174939), errback=(continuation of Deferred@342395714 after retry RPC after error@846627864) -> (continuation of Deferred@326817581 after retry RPC after error@898467843) -> (continuation of Deferred@1701783403 after retry RPC after error@1665083802) -> (continuation of Deferred@649677167 after retry RPC after error@2133655595) -> (continuation of Deferred@1675965658 after retry RPC after error@2039263941) -> (continuation of Deferred@353533610 after retry RPC after error@44632187) -> (continuation of Deferred@1218494224 after retry RPC after error@1420174939)))
03:00:13.162 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@972289117(state=PENDING, result=null, callback=(continuation of Deferred@342395714 after retry RPC after error@846627864) -> (continuation of Deferred@326817581 after retry RPC after error@898467843) -> (continuation of Deferred@1701783403 after retry RPC after error@1665083802) -> (continuation of Deferred@649677167 after retry RPC after error@2133655595) -> (continuation of Deferred@1675965658 after retry RPC after error@2039263941) -> (continuation of Deferred@353533610 after retry RPC after error@44632187) -> (continuation of Deferred@1218494224 after retry RPC after error@1420174939), errback=(continuation of Deferred@342395714 after retry RPC after error@846627864) -> (continuation of Deferred@326817581 after retry RPC after error@898467843) -> (continuation of Deferred@1701783403 after retry RPC after error@1665083802) -> (continuation of Deferred@649677167 after retry RPC after error@2133655595) -> (continuation of Deferred@1675965658 after retry RPC after error@2039263941) -> (continuation of Deferred@353533610 after retry RPC after error@44632187) -> (continuation of Deferred@1218494224 after retry RPC after error@1420174939)))
03:00:13.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.180 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.180 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.180 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.180 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.180 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.180 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.199 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.200 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.200 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.200 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.200 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.200 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.200 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.221 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.221 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.221 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.221 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.221 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.221 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.221 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.222 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.241 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.241 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.241 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.241 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.241 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.241 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.241 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.242 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.261 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.261 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.261 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.261 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.261 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.261 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.262 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.281 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.281 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.281 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.281 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.281 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.281 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.282 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.300 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.301 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.301 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.301 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.301 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.301 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.302 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.379 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.379 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.380 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.380 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.380 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.380 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.380 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.380 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.381 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1560479590(state=PENDING, result=null, callback=(continuation of Deferred@2070926203 after retry RPC after error@995216244) -> (continuation of Deferred@359694927 after retry RPC after error@1351454640) -> (continuation of Deferred@1434217169 after retry RPC after error@1615008097) -> (continuation of Deferred@1907398108 after retry RPC after error@1308386389) -> (continuation of Deferred@2078324185 after retry RPC after error@63994845) -> (continuation of Deferred@1983582222 after retry RPC after error@1489706064) -> (continuation of Deferred@645608685 after retry RPC after error@653634924), errback=(continuation of Deferred@2070926203 after retry RPC after error@995216244) -> (continuation of Deferred@359694927 after retry RPC after error@1351454640) -> (continuation of Deferred@1434217169 after retry RPC after error@1615008097) -> (continuation of Deferred@1907398108 after retry RPC after error@1308386389) -> (continuation of Deferred@2078324185 after retry RPC after error@63994845) -> (continuation of Deferred@1983582222 after retry RPC after error@1489706064) -> (continuation of Deferred@645608685 after retry RPC after error@653634924)))
03:00:13.381 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1560479590(state=PENDING, result=null, callback=(continuation of Deferred@2070926203 after retry RPC after error@995216244) -> (continuation of Deferred@359694927 after retry RPC after error@1351454640) -> (continuation of Deferred@1434217169 after retry RPC after error@1615008097) -> (continuation of Deferred@1907398108 after retry RPC after error@1308386389) -> (continuation of Deferred@2078324185 after retry RPC after error@63994845) -> (continuation of Deferred@1983582222 after retry RPC after error@1489706064) -> (continuation of Deferred@645608685 after retry RPC after error@653634924), errback=(continuation of Deferred@2070926203 after retry RPC after error@995216244) -> (continuation of Deferred@359694927 after retry RPC after error@1351454640) -> (continuation of Deferred@1434217169 after retry RPC after error@1615008097) -> (continuation of Deferred@1907398108 after retry RPC after error@1308386389) -> (continuation of Deferred@2078324185 after retry RPC after error@63994845) -> (continuation of Deferred@1983582222 after retry RPC after error@1489706064) -> (continuation of Deferred@645608685 after retry RPC after error@653634924)))
03:00:13.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.401 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.401 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.401 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.401 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.401 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.401 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.402 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.421 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.421 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.421 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.421 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.421 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.421 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.422 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.440 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.440 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.440 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.440 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.440 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.440 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.459 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.460 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.460 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.460 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.460 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.460 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.460 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.461 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.479 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.480 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.480 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.480 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.480 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.481 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.481 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.481 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.501 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.501 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.501 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.501 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.501 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.501 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.502 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.520 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.521 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.521 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.521 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.521 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.521 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.522 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.581 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.581 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.581 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.581 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.581 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.581 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.582 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.582 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2005066041(state=PENDING, result=null, callback=(continuation of Deferred@826093811 after retry RPC after error@340723335) -> (continuation of Deferred@274953900 after retry RPC after error@1888011952) -> (continuation of Deferred@686476026 after retry RPC after error@1149078110) -> (continuation of Deferred@1327518315 after retry RPC after error@2032902265) -> (continuation of Deferred@132651196 after retry RPC after error@804216165) -> (continuation of Deferred@1645343225 after retry RPC after error@1184738730) -> (continuation of Deferred@24604733 after retry RPC after error@1420169475), errback=(continuation of Deferred@826093811 after retry RPC after error@340723335) -> (continuation of Deferred@274953900 after retry RPC after error@1888011952) -> (continuation of Deferred@686476026 after retry RPC after error@1149078110) -> (continuation of Deferred@1327518315 after retry RPC after error@2032902265) -> (continuation of Deferred@132651196 after retry RPC after error@804216165) -> (continuation of Deferred@1645343225 after retry RPC after error@1184738730) -> (continuation of Deferred@24604733 after retry RPC after error@1420169475)))
03:00:13.583 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2005066041(state=PENDING, result=null, callback=(continuation of Deferred@826093811 after retry RPC after error@340723335) -> (continuation of Deferred@274953900 after retry RPC after error@1888011952) -> (continuation of Deferred@686476026 after retry RPC after error@1149078110) -> (continuation of Deferred@1327518315 after retry RPC after error@2032902265) -> (continuation of Deferred@132651196 after retry RPC after error@804216165) -> (continuation of Deferred@1645343225 after retry RPC after error@1184738730) -> (continuation of Deferred@24604733 after retry RPC after error@1420169475), errback=(continuation of Deferred@826093811 after retry RPC after error@340723335) -> (continuation of Deferred@274953900 after retry RPC after error@1888011952) -> (continuation of Deferred@686476026 after retry RPC after error@1149078110) -> (continuation of Deferred@1327518315 after retry RPC after error@2032902265) -> (continuation of Deferred@132651196 after retry RPC after error@804216165) -> (continuation of Deferred@1645343225 after retry RPC after error@1184738730) -> (continuation of Deferred@24604733 after retry RPC after error@1420169475)))
03:00:13.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.601 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.601 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.601 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.601 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.601 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.601 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.621 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.621 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.621 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.621 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.621 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.621 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.622 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.641 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.641 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.641 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.641 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.641 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.641 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.641 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.661 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.661 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.661 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.661 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.661 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.661 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.680 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.680 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.680 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.680 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.681 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.681 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.700 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.700 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.700 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.700 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.700 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.700 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.740 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.740 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.740 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.740 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.740 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.740 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.841 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.841 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.841 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.841 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.841 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.841 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.842 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.842 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1487622686(state=PENDING, result=null, callback=(continuation of Deferred@1072890055 after retry RPC after error@31013197) -> (continuation of Deferred@169635762 after retry RPC after error@136701773) -> (continuation of Deferred@804731830 after retry RPC after error@1730964564) -> (continuation of Deferred@237305596 after retry RPC after error@698118138) -> (continuation of Deferred@1067506676 after retry RPC after error@1005211288) -> (continuation of Deferred@303076359 after retry RPC after error@1488574976) -> (continuation of Deferred@1635763608 after retry RPC after error@134797729), errback=(continuation of Deferred@1072890055 after retry RPC after error@31013197) -> (continuation of Deferred@169635762 after retry RPC after error@136701773) -> (continuation of Deferred@804731830 after retry RPC after error@1730964564) -> (continuation of Deferred@237305596 after retry RPC after error@698118138) -> (continuation of Deferred@1067506676 after retry RPC after error@1005211288) -> (continuation of Deferred@303076359 after retry RPC after error@1488574976) -> (continuation of Deferred@1635763608 after retry RPC after error@134797729)))
03:00:13.842 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1487622686(state=PENDING, result=null, callback=(continuation of Deferred@1072890055 after retry RPC after error@31013197) -> (continuation of Deferred@169635762 after retry RPC after error@136701773) -> (continuation of Deferred@804731830 after retry RPC after error@1730964564) -> (continuation of Deferred@237305596 after retry RPC after error@698118138) -> (continuation of Deferred@1067506676 after retry RPC after error@1005211288) -> (continuation of Deferred@303076359 after retry RPC after error@1488574976) -> (continuation of Deferred@1635763608 after retry RPC after error@134797729), errback=(continuation of Deferred@1072890055 after retry RPC after error@31013197) -> (continuation of Deferred@169635762 after retry RPC after error@136701773) -> (continuation of Deferred@804731830 after retry RPC after error@1730964564) -> (continuation of Deferred@237305596 after retry RPC after error@698118138) -> (continuation of Deferred@1067506676 after retry RPC after error@1005211288) -> (continuation of Deferred@303076359 after retry RPC after error@1488574976) -> (continuation of Deferred@1635763608 after retry RPC after error@134797729)))
03:00:13.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.860 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.861 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.861 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.861 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.861 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.861 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.880 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.880 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.880 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.880 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.880 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.881 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.899 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.900 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.900 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.900 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.900 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.900 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.900 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.921 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.921 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.921 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.921 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.921 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.921 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.922 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.922 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.941 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.941 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.941 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.941 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.941 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.941 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.942 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:13.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:13.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:13.981 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.981 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.981 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:13.981 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.981 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.981 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:13.982 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.021 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.021 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.021 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.021 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.021 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.021 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.022 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.022 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@837860533(state=PENDING, result=null, callback=(continuation of Deferred@847421403 after retry RPC after error@1291662115) -> (continuation of Deferred@615231753 after retry RPC after error@269132861) -> (continuation of Deferred@911252896 after retry RPC after error@319935586) -> (continuation of Deferred@1791692016 after retry RPC after error@799373016) -> (continuation of Deferred@497697553 after retry RPC after error@8730114) -> (continuation of Deferred@849020526 after retry RPC after error@906798419), errback=(continuation of Deferred@847421403 after retry RPC after error@1291662115) -> (continuation of Deferred@615231753 after retry RPC after error@269132861) -> (continuation of Deferred@911252896 after retry RPC after error@319935586) -> (continuation of Deferred@1791692016 after retry RPC after error@799373016) -> (continuation of Deferred@497697553 after retry RPC after error@8730114) -> (continuation of Deferred@849020526 after retry RPC after error@906798419)))
03:00:14.022 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@837860533(state=PENDING, result=null, callback=(continuation of Deferred@847421403 after retry RPC after error@1291662115) -> (continuation of Deferred@615231753 after retry RPC after error@269132861) -> (continuation of Deferred@911252896 after retry RPC after error@319935586) -> (continuation of Deferred@1791692016 after retry RPC after error@799373016) -> (continuation of Deferred@497697553 after retry RPC after error@8730114) -> (continuation of Deferred@849020526 after retry RPC after error@906798419), errback=(continuation of Deferred@847421403 after retry RPC after error@1291662115) -> (continuation of Deferred@615231753 after retry RPC after error@269132861) -> (continuation of Deferred@911252896 after retry RPC after error@319935586) -> (continuation of Deferred@1791692016 after retry RPC after error@799373016) -> (continuation of Deferred@497697553 after retry RPC after error@8730114) -> (continuation of Deferred@849020526 after retry RPC after error@906798419)))
03:00:14.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.041 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.041 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.041 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.041 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.041 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.041 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.042 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.061 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.061 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.061 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.061 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.061 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.061 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.081 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.081 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.081 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.081 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.081 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.081 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.082 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.101 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.101 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.101 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.101 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.101 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.101 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.121 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.121 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.121 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.121 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.121 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.121 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.161 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.161 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.161 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.161 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.161 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.161 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.180 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.180 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.180 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.180 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.180 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.180 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.261 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.261 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.261 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.262 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.262 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.262 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.262 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.262 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1904956581(state=PENDING, result=null, callback=(continuation of Deferred@436947867 after retry RPC after error@1579157970) -> (continuation of Deferred@27898509 after retry RPC after error@707584097) -> (continuation of Deferred@1790652716 after retry RPC after error@873188144) -> (continuation of Deferred@2021276390 after retry RPC after error@899211872) -> (continuation of Deferred@102336430 after retry RPC after error@1822526998) -> (continuation of Deferred@1160062062 after retry RPC after error@928793928) -> (continuation of Deferred@2100461818 after retry RPC after error@495153268), errback=(continuation of Deferred@436947867 after retry RPC after error@1579157970) -> (continuation of Deferred@27898509 after retry RPC after error@707584097) -> (continuation of Deferred@1790652716 after retry RPC after error@873188144) -> (continuation of Deferred@2021276390 after retry RPC after error@899211872) -> (continuation of Deferred@102336430 after retry RPC after error@1822526998) -> (continuation of Deferred@1160062062 after retry RPC after error@928793928) -> (continuation of Deferred@2100461818 after retry RPC after error@495153268)))
03:00:14.262 [DEBUG - kudu-nio-2] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1904956581(state=PENDING, result=null, callback=(continuation of Deferred@436947867 after retry RPC after error@1579157970) -> (continuation of Deferred@27898509 after retry RPC after error@707584097) -> (continuation of Deferred@1790652716 after retry RPC after error@873188144) -> (continuation of Deferred@2021276390 after retry RPC after error@899211872) -> (continuation of Deferred@102336430 after retry RPC after error@1822526998) -> (continuation of Deferred@1160062062 after retry RPC after error@928793928) -> (continuation of Deferred@2100461818 after retry RPC after error@495153268), errback=(continuation of Deferred@436947867 after retry RPC after error@1579157970) -> (continuation of Deferred@27898509 after retry RPC after error@707584097) -> (continuation of Deferred@1790652716 after retry RPC after error@873188144) -> (continuation of Deferred@2021276390 after retry RPC after error@899211872) -> (continuation of Deferred@102336430 after retry RPC after error@1822526998) -> (continuation of Deferred@1160062062 after retry RPC after error@928793928) -> (continuation of Deferred@2100461818 after retry RPC after error@495153268)))
03:00:14.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.281 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.281 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.281 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.281 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.281 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.281 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.282 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.300 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.300 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.300 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.301 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.301 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.301 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.320 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.320 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.320 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.320 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.321 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.321 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.339 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.340 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.340 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.340 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.340 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.340 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.340 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.359 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.359 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.360 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.360 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.360 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.360 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.360 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.360 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.401 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.401 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.401 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.401 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.401 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.401 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.402 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.461 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.461 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.461 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.461 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.461 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.461 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.461 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.461 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.462 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@875932439(state=PENDING, result=null, callback=(continuation of Deferred@1837232241 after retry RPC after error@183490514) -> (continuation of Deferred@1450245383 after retry RPC after error@1334086093) -> (continuation of Deferred@769976884 after retry RPC after error@1505880349) -> (continuation of Deferred@143175655 after retry RPC after error@1672054234) -> (continuation of Deferred@1771464401 after retry RPC after error@1070195711) -> (continuation of Deferred@1920177270 after retry RPC after error@1566267575), errback=(continuation of Deferred@1837232241 after retry RPC after error@183490514) -> (continuation of Deferred@1450245383 after retry RPC after error@1334086093) -> (continuation of Deferred@769976884 after retry RPC after error@1505880349) -> (continuation of Deferred@143175655 after retry RPC after error@1672054234) -> (continuation of Deferred@1771464401 after retry RPC after error@1070195711) -> (continuation of Deferred@1920177270 after retry RPC after error@1566267575)))
03:00:14.462 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@875932439(state=PENDING, result=null, callback=(continuation of Deferred@1837232241 after retry RPC after error@183490514) -> (continuation of Deferred@1450245383 after retry RPC after error@1334086093) -> (continuation of Deferred@769976884 after retry RPC after error@1505880349) -> (continuation of Deferred@143175655 after retry RPC after error@1672054234) -> (continuation of Deferred@1771464401 after retry RPC after error@1070195711) -> (continuation of Deferred@1920177270 after retry RPC after error@1566267575), errback=(continuation of Deferred@1837232241 after retry RPC after error@183490514) -> (continuation of Deferred@1450245383 after retry RPC after error@1334086093) -> (continuation of Deferred@769976884 after retry RPC after error@1505880349) -> (continuation of Deferred@143175655 after retry RPC after error@1672054234) -> (continuation of Deferred@1771464401 after retry RPC after error@1070195711) -> (continuation of Deferred@1920177270 after retry RPC after error@1566267575)))
03:00:14.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.481 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.481 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.481 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.481 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.481 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.481 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.481 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.482 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.499 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.500 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.500 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.500 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.500 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.500 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.500 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.519 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.520 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.520 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.520 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.520 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.520 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.520 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.539 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.540 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.540 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.540 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.540 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.540 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.540 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.561 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.561 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.561 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.561 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.561 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.561 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.561 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.562 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.599 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.600 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.600 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.600 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.600 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.600 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.600 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.600 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.639 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.640 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.640 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.640 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.640 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.640 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.640 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.641 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@431569553(state=PENDING, result=null, callback=(continuation of Deferred@513820657 after retry RPC after error@300390659) -> (continuation of Deferred@1487422916 after retry RPC after error@135631346) -> (continuation of Deferred@337039566 after retry RPC after error@957643982) -> (continuation of Deferred@596154445 after retry RPC after error@120981455) -> (continuation of Deferred@621033287 after retry RPC after error@1064099109) -> (continuation of Deferred@1480679541 after retry RPC after error@1878924493), errback=(continuation of Deferred@513820657 after retry RPC after error@300390659) -> (continuation of Deferred@1487422916 after retry RPC after error@135631346) -> (continuation of Deferred@337039566 after retry RPC after error@957643982) -> (continuation of Deferred@596154445 after retry RPC after error@120981455) -> (continuation of Deferred@621033287 after retry RPC after error@1064099109) -> (continuation of Deferred@1480679541 after retry RPC after error@1878924493)))
03:00:14.641 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@431569553(state=PENDING, result=null, callback=(continuation of Deferred@513820657 after retry RPC after error@300390659) -> (continuation of Deferred@1487422916 after retry RPC after error@135631346) -> (continuation of Deferred@337039566 after retry RPC after error@957643982) -> (continuation of Deferred@596154445 after retry RPC after error@120981455) -> (continuation of Deferred@621033287 after retry RPC after error@1064099109) -> (continuation of Deferred@1480679541 after retry RPC after error@1878924493), errback=(continuation of Deferred@513820657 after retry RPC after error@300390659) -> (continuation of Deferred@1487422916 after retry RPC after error@135631346) -> (continuation of Deferred@337039566 after retry RPC after error@957643982) -> (continuation of Deferred@596154445 after retry RPC after error@120981455) -> (continuation of Deferred@621033287 after retry RPC after error@1064099109) -> (continuation of Deferred@1480679541 after retry RPC after error@1878924493)))
03:00:14.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.661 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.661 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.661 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.661 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.661 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.661 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.662 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.681 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.681 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.681 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.681 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.681 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.681 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.682 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.701 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.701 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.701 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.701 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.701 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.701 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.702 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.721 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.721 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.721 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.721 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.721 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.721 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.721 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.740 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.741 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.741 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.741 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.741 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.741 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.760 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.760 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.760 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.760 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.760 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.761 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.800 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.800 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.800 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.800 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.800 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.800 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.901 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.901 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.901 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.901 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.901 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.901 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.902 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.902 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1873681789(state=PENDING, result=null, callback=(continuation of Deferred@1621263037 after retry RPC after error@1759945655) -> (continuation of Deferred@94976111 after retry RPC after error@443779154) -> (continuation of Deferred@1848657611 after retry RPC after error@857123745) -> (continuation of Deferred@2133891052 after retry RPC after error@2030543168) -> (continuation of Deferred@367896245 after retry RPC after error@329272330) -> (continuation of Deferred@1750349413 after retry RPC after error@748525454) -> (continuation of Deferred@1413305510 after retry RPC after error@1173131831), errback=(continuation of Deferred@1621263037 after retry RPC after error@1759945655) -> (continuation of Deferred@94976111 after retry RPC after error@443779154) -> (continuation of Deferred@1848657611 after retry RPC after error@857123745) -> (continuation of Deferred@2133891052 after retry RPC after error@2030543168) -> (continuation of Deferred@367896245 after retry RPC after error@329272330) -> (continuation of Deferred@1750349413 after retry RPC after error@748525454) -> (continuation of Deferred@1413305510 after retry RPC after error@1173131831)))
03:00:14.902 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1873681789(state=PENDING, result=null, callback=(continuation of Deferred@1621263037 after retry RPC after error@1759945655) -> (continuation of Deferred@94976111 after retry RPC after error@443779154) -> (continuation of Deferred@1848657611 after retry RPC after error@857123745) -> (continuation of Deferred@2133891052 after retry RPC after error@2030543168) -> (continuation of Deferred@367896245 after retry RPC after error@329272330) -> (continuation of Deferred@1750349413 after retry RPC after error@748525454) -> (continuation of Deferred@1413305510 after retry RPC after error@1173131831), errback=(continuation of Deferred@1621263037 after retry RPC after error@1759945655) -> (continuation of Deferred@94976111 after retry RPC after error@443779154) -> (continuation of Deferred@1848657611 after retry RPC after error@857123745) -> (continuation of Deferred@2133891052 after retry RPC after error@2030543168) -> (continuation of Deferred@367896245 after retry RPC after error@329272330) -> (continuation of Deferred@1750349413 after retry RPC after error@748525454) -> (continuation of Deferred@1413305510 after retry RPC after error@1173131831)))
03:00:14.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.920 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.921 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.921 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.921 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.921 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.921 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.940 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.940 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.941 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.941 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.941 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.941 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.960 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.960 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.960 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.960 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.960 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.960 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:14.979 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:14.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:14.980 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.980 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.980 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.980 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:14.980 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.980 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:14.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.001 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.001 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.001 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.001 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.001 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.001 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.002 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.041 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.041 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.041 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.041 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.041 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.041 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.042 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.061 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.061 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.061 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.061 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.061 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.061 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.062 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.080 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.081 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.081 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.081 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.081 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.081 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.081 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.152 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.154 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.154 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.154 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.155 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.155 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.155 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.155 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.156 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.156 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=236), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [234ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [234ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [235ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [236ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1587324603(state=PENDING, result=null, callback=(continuation of Deferred@1328902502 after retry RPC after error@444272959) -> (continuation of Deferred@686639564 after retry RPC after error@327129876) -> (continuation of Deferred@2007328348 after retry RPC after error@898986878) -> (continuation of Deferred@2004152105 after retry RPC after error@406613480) -> (continuation of Deferred@1157070645 after retry RPC after error@1525152342) -> (continuation of Deferred@179985284 after retry RPC after error@1897014294) -> (continuation of Deferred@249816022 after retry RPC after error@1806933270) -> (continuation of Deferred@856526239 after retry RPC after error@1398750588), errback=(continuation of Deferred@1328902502 after retry RPC after error@444272959) -> (continuation of Deferred@686639564 after retry RPC after error@327129876) -> (continuation of Deferred@2007328348 after retry RPC after error@898986878) -> (continuation of Deferred@2004152105 after retry RPC after error@406613480) -> (continuation of Deferred@1157070645 after retry RPC after error@1525152342) -> (continuation of Deferred@179985284 after retry RPC after error@1897014294) -> (continuation of Deferred@249816022 after retry RPC after error@1806933270) -> (continuation of Deferred@856526239 after retry RPC after error@1398750588)))
03:00:15.156 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=236), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [234ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [234ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [235ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [236ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1587324603(state=PENDING, result=null, callback=(continuation of Deferred@1328902502 after retry RPC after error@444272959) -> (continuation of Deferred@686639564 after retry RPC after error@327129876) -> (continuation of Deferred@2007328348 after retry RPC after error@898986878) -> (continuation of Deferred@2004152105 after retry RPC after error@406613480) -> (continuation of Deferred@1157070645 after retry RPC after error@1525152342) -> (continuation of Deferred@179985284 after retry RPC after error@1897014294) -> (continuation of Deferred@249816022 after retry RPC after error@1806933270) -> (continuation of Deferred@856526239 after retry RPC after error@1398750588), errback=(continuation of Deferred@1328902502 after retry RPC after error@444272959) -> (continuation of Deferred@686639564 after retry RPC after error@327129876) -> (continuation of Deferred@2007328348 after retry RPC after error@898986878) -> (continuation of Deferred@2004152105 after retry RPC after error@406613480) -> (continuation of Deferred@1157070645 after retry RPC after error@1525152342) -> (continuation of Deferred@179985284 after retry RPC after error@1897014294) -> (continuation of Deferred@249816022 after retry RPC after error@1806933270) -> (continuation of Deferred@856526239 after retry RPC after error@1398750588)))
03:00:15.179 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.179 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.180 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.180 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.180 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.180 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.180 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.180 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.201 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.201 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.201 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.201 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.201 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.201 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.220 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.220 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.220 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.221 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.221 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.221 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.240 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.240 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.240 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.240 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.241 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.241 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.241 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.259 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.260 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.260 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.260 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.260 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.260 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.260 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.261 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.299 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.300 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.300 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.300 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.300 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.300 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.300 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.320 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.320 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.320 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.320 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.321 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.321 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.322 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.361 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.361 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.361 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.361 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.361 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.361 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.362 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.381 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.381 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.381 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.381 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.381 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.381 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.381 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.382 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.382 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [203ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@513703887(state=PENDING, result=null, callback=(continuation of Deferred@1416909820 after retry RPC after error@832376961) -> (continuation of Deferred@227846760 after retry RPC after error@1620696082) -> (continuation of Deferred@377788494 after retry RPC after error@1338284807) -> (continuation of Deferred@1730920317 after retry RPC after error@1232897734) -> (continuation of Deferred@2064130383 after retry RPC after error@276795786) -> (continuation of Deferred@714413231 after retry RPC after error@124972429) -> (continuation of Deferred@1028302453 after retry RPC after error@1364558743) -> (continuation of Deferred@1407702398 after retry RPC after error@1139364874), errback=(continuation of Deferred@1416909820 after retry RPC after error@832376961) -> (continuation of Deferred@227846760 after retry RPC after error@1620696082) -> (continuation of Deferred@377788494 after retry RPC after error@1338284807) -> (continuation of Deferred@1730920317 after retry RPC after error@1232897734) -> (continuation of Deferred@2064130383 after retry RPC after error@276795786) -> (continuation of Deferred@714413231 after retry RPC after error@124972429) -> (continuation of Deferred@1028302453 after retry RPC after error@1364558743) -> (continuation of Deferred@1407702398 after retry RPC after error@1139364874)))
03:00:15.382 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [183ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [203ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@513703887(state=PENDING, result=null, callback=(continuation of Deferred@1416909820 after retry RPC after error@832376961) -> (continuation of Deferred@227846760 after retry RPC after error@1620696082) -> (continuation of Deferred@377788494 after retry RPC after error@1338284807) -> (continuation of Deferred@1730920317 after retry RPC after error@1232897734) -> (continuation of Deferred@2064130383 after retry RPC after error@276795786) -> (continuation of Deferred@714413231 after retry RPC after error@124972429) -> (continuation of Deferred@1028302453 after retry RPC after error@1364558743) -> (continuation of Deferred@1407702398 after retry RPC after error@1139364874), errback=(continuation of Deferred@1416909820 after retry RPC after error@832376961) -> (continuation of Deferred@227846760 after retry RPC after error@1620696082) -> (continuation of Deferred@377788494 after retry RPC after error@1338284807) -> (continuation of Deferred@1730920317 after retry RPC after error@1232897734) -> (continuation of Deferred@2064130383 after retry RPC after error@276795786) -> (continuation of Deferred@714413231 after retry RPC after error@124972429) -> (continuation of Deferred@1028302453 after retry RPC after error@1364558743) -> (continuation of Deferred@1407702398 after retry RPC after error@1139364874)))
03:00:15.399 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.400 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.400 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.400 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.400 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.400 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.400 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.421 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.421 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.421 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.421 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.421 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.421 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.421 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.421 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.422 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.422 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.441 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.441 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.441 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.441 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.441 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.441 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.460 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.460 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.460 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.460 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.461 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.461 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.461 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.480 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.480 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.480 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.480 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.480 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.480 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.482 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.499 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.500 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.500 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.500 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.500 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.500 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.500 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.519 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.520 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.520 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.520 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.520 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.520 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.520 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.640 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.641 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.641 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.641 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.641 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.641 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.642 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] refreshing cache from master, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1491572780(state=PENDING, result=null, callback=(continuation of Deferred@1828261323 after retry RPC after error@1044172878) -> (continuation of Deferred@1742849698 after retry RPC after error@61019274) -> (continuation of Deferred@655414880 after retry RPC after error@2146880620) -> (continuation of Deferred@374063590 after retry RPC after error@223838433) -> (continuation of Deferred@743830781 after retry RPC after error@1695144119) -> (continuation of Deferred@755327233 after retry RPC after error@484752584) -> (continuation of Deferred@521486773 after retry RPC after error@888340096), errback=(continuation of Deferred@1828261323 after retry RPC after error@1044172878) -> (continuation of Deferred@1742849698 after retry RPC after error@61019274) -> (continuation of Deferred@655414880 after retry RPC after error@2146880620) -> (continuation of Deferred@374063590 after retry RPC after error@223838433) -> (continuation of Deferred@743830781 after retry RPC after error@1695144119) -> (continuation of Deferred@755327233 after retry RPC after error@484752584) -> (continuation of Deferred@521486773 after retry RPC after error@888340096)))
03:00:15.642 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] refreshing cache from master, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1491572780(state=PENDING, result=null, callback=(continuation of Deferred@1828261323 after retry RPC after error@1044172878) -> (continuation of Deferred@1742849698 after retry RPC after error@61019274) -> (continuation of Deferred@655414880 after retry RPC after error@2146880620) -> (continuation of Deferred@374063590 after retry RPC after error@223838433) -> (continuation of Deferred@743830781 after retry RPC after error@1695144119) -> (continuation of Deferred@755327233 after retry RPC after error@484752584) -> (continuation of Deferred@521486773 after retry RPC after error@888340096), errback=(continuation of Deferred@1828261323 after retry RPC after error@1044172878) -> (continuation of Deferred@1742849698 after retry RPC after error@61019274) -> (continuation of Deferred@655414880 after retry RPC after error@2146880620) -> (continuation of Deferred@374063590 after retry RPC after error@223838433) -> (continuation of Deferred@743830781 after retry RPC after error@1695144119) -> (continuation of Deferred@755327233 after retry RPC after error@484752584) -> (continuation of Deferred@521486773 after retry RPC after error@888340096)))
03:00:15.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.660 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.660 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.660 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.660 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.660 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.660 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.679 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.679 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.680 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.680 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.680 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.680 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.680 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.680 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.701 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.701 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.701 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.701 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.701 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.701 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.702 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.720 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.720 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.720 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.720 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.720 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.721 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.721 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.739 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.740 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.740 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.740 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.740 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.740 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.740 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.779 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.780 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.780 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.780 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.780 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.780 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.780 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.839 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.840 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.840 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.840 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.840 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.840 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.840 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.841 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1672400505(state=PENDING, result=null, callback=(continuation of Deferred@2077502621 after retry RPC after error@962081042) -> (continuation of Deferred@1996528408 after retry RPC after error@1657294471) -> (continuation of Deferred@950154887 after retry RPC after error@1082806304) -> (continuation of Deferred@857856882 after retry RPC after error@331159527) -> (continuation of Deferred@973212953 after retry RPC after error@672874793) -> (continuation of Deferred@366798099 after retry RPC after error@1847974496), errback=(continuation of Deferred@2077502621 after retry RPC after error@962081042) -> (continuation of Deferred@1996528408 after retry RPC after error@1657294471) -> (continuation of Deferred@950154887 after retry RPC after error@1082806304) -> (continuation of Deferred@857856882 after retry RPC after error@331159527) -> (continuation of Deferred@973212953 after retry RPC after error@672874793) -> (continuation of Deferred@366798099 after retry RPC after error@1847974496)))
03:00:15.842 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1672400505(state=PENDING, result=null, callback=(continuation of Deferred@2077502621 after retry RPC after error@962081042) -> (continuation of Deferred@1996528408 after retry RPC after error@1657294471) -> (continuation of Deferred@950154887 after retry RPC after error@1082806304) -> (continuation of Deferred@857856882 after retry RPC after error@331159527) -> (continuation of Deferred@973212953 after retry RPC after error@672874793) -> (continuation of Deferred@366798099 after retry RPC after error@1847974496), errback=(continuation of Deferred@2077502621 after retry RPC after error@962081042) -> (continuation of Deferred@1996528408 after retry RPC after error@1657294471) -> (continuation of Deferred@950154887 after retry RPC after error@1082806304) -> (continuation of Deferred@857856882 after retry RPC after error@331159527) -> (continuation of Deferred@973212953 after retry RPC after error@672874793) -> (continuation of Deferred@366798099 after retry RPC after error@1847974496)))
03:00:15.859 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.859 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.860 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.860 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.860 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.860 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.860 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.860 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.880 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.881 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.881 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.881 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.881 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.881 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.882 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.900 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.901 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.901 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.901 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.901 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.901 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.920 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.920 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.920 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.920 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.920 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.921 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.939 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.940 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.940 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.940 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.940 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.940 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.940 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:15.979 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:15.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:15.980 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.980 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.980 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.980 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.980 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:15.980 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:15.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.039 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.040 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.040 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.040 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.040 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.041 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.041 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.041 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.041 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1141105786(state=PENDING, result=null, callback=(continuation of Deferred@79024192 after retry RPC after error@1803312662) -> (continuation of Deferred@1991922662 after retry RPC after error@759257471) -> (continuation of Deferred@123517767 after retry RPC after error@240769364) -> (continuation of Deferred@710446618 after retry RPC after error@234205327) -> (continuation of Deferred@1871815925 after retry RPC after error@792871722) -> (continuation of Deferred@1090793606 after retry RPC after error@1632993761), errback=(continuation of Deferred@79024192 after retry RPC after error@1803312662) -> (continuation of Deferred@1991922662 after retry RPC after error@759257471) -> (continuation of Deferred@123517767 after retry RPC after error@240769364) -> (continuation of Deferred@710446618 after retry RPC after error@234205327) -> (continuation of Deferred@1871815925 after retry RPC after error@792871722) -> (continuation of Deferred@1090793606 after retry RPC after error@1632993761)))
03:00:16.042 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1141105786(state=PENDING, result=null, callback=(continuation of Deferred@79024192 after retry RPC after error@1803312662) -> (continuation of Deferred@1991922662 after retry RPC after error@759257471) -> (continuation of Deferred@123517767 after retry RPC after error@240769364) -> (continuation of Deferred@710446618 after retry RPC after error@234205327) -> (continuation of Deferred@1871815925 after retry RPC after error@792871722) -> (continuation of Deferred@1090793606 after retry RPC after error@1632993761), errback=(continuation of Deferred@79024192 after retry RPC after error@1803312662) -> (continuation of Deferred@1991922662 after retry RPC after error@759257471) -> (continuation of Deferred@123517767 after retry RPC after error@240769364) -> (continuation of Deferred@710446618 after retry RPC after error@234205327) -> (continuation of Deferred@1871815925 after retry RPC after error@792871722) -> (continuation of Deferred@1090793606 after retry RPC after error@1632993761)))
03:00:16.059 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.060 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.060 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.060 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.061 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.061 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.061 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.081 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.081 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.081 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.081 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.081 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.081 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.081 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.082 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.101 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.101 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.101 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.101 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.101 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.101 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.120 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.121 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.121 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.121 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.121 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.121 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.140 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.140 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.141 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.141 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.141 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.141 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.141 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.160 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.160 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.160 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.161 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.161 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.161 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.200 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.200 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.201 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.201 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.201 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.201 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.261 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.261 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.261 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.261 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.261 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.261 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.262 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.262 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [203ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@421670254(state=PENDING, result=null, callback=(continuation of Deferred@482022678 after retry RPC after error@2099028048) -> (continuation of Deferred@1864029074 after retry RPC after error@1742457410) -> (continuation of Deferred@588013578 after retry RPC after error@990534171) -> (continuation of Deferred@37066923 after retry RPC after error@998628213) -> (continuation of Deferred@2076831807 after retry RPC after error@310032351) -> (continuation of Deferred@1017419259 after retry RPC after error@695185257) -> (continuation of Deferred@496233075 after retry RPC after error@1511027281), errback=(continuation of Deferred@482022678 after retry RPC after error@2099028048) -> (continuation of Deferred@1864029074 after retry RPC after error@1742457410) -> (continuation of Deferred@588013578 after retry RPC after error@990534171) -> (continuation of Deferred@37066923 after retry RPC after error@998628213) -> (continuation of Deferred@2076831807 after retry RPC after error@310032351) -> (continuation of Deferred@1017419259 after retry RPC after error@695185257) -> (continuation of Deferred@496233075 after retry RPC after error@1511027281)))
03:00:16.262 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [203ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@421670254(state=PENDING, result=null, callback=(continuation of Deferred@482022678 after retry RPC after error@2099028048) -> (continuation of Deferred@1864029074 after retry RPC after error@1742457410) -> (continuation of Deferred@588013578 after retry RPC after error@990534171) -> (continuation of Deferred@37066923 after retry RPC after error@998628213) -> (continuation of Deferred@2076831807 after retry RPC after error@310032351) -> (continuation of Deferred@1017419259 after retry RPC after error@695185257) -> (continuation of Deferred@496233075 after retry RPC after error@1511027281), errback=(continuation of Deferred@482022678 after retry RPC after error@2099028048) -> (continuation of Deferred@1864029074 after retry RPC after error@1742457410) -> (continuation of Deferred@588013578 after retry RPC after error@990534171) -> (continuation of Deferred@37066923 after retry RPC after error@998628213) -> (continuation of Deferred@2076831807 after retry RPC after error@310032351) -> (continuation of Deferred@1017419259 after retry RPC after error@695185257) -> (continuation of Deferred@496233075 after retry RPC after error@1511027281)))
03:00:16.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.280 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.280 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.280 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.281 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.281 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.281 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.300 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.300 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.300 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.300 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.301 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.301 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.320 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.321 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.321 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.321 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.321 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.321 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.340 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.340 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.340 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.340 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.340 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.341 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.360 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.360 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.360 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.360 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.361 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.361 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.362 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.379 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.379 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.380 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.380 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.380 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.380 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.380 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.380 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.459 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.460 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.460 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.460 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.460 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.460 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.460 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.461 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.499 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.500 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.500 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.500 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.500 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.500 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.500 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.501 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@377482896(state=PENDING, result=null, callback=(continuation of Deferred@504856616 after retry RPC after error@19714876) -> (continuation of Deferred@480226246 after retry RPC after error@1971183019) -> (continuation of Deferred@1472792044 after retry RPC after error@807115034) -> (continuation of Deferred@1371264324 after retry RPC after error@1253104221) -> (continuation of Deferred@1439256706 after retry RPC after error@964087912) -> (continuation of Deferred@912794615 after retry RPC after error@2059822421) -> (continuation of Deferred@1695836111 after retry RPC after error@1793990293), errback=(continuation of Deferred@504856616 after retry RPC after error@19714876) -> (continuation of Deferred@480226246 after retry RPC after error@1971183019) -> (continuation of Deferred@1472792044 after retry RPC after error@807115034) -> (continuation of Deferred@1371264324 after retry RPC after error@1253104221) -> (continuation of Deferred@1439256706 after retry RPC after error@964087912) -> (continuation of Deferred@912794615 after retry RPC after error@2059822421) -> (continuation of Deferred@1695836111 after retry RPC after error@1793990293)))
03:00:16.502 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@377482896(state=PENDING, result=null, callback=(continuation of Deferred@504856616 after retry RPC after error@19714876) -> (continuation of Deferred@480226246 after retry RPC after error@1971183019) -> (continuation of Deferred@1472792044 after retry RPC after error@807115034) -> (continuation of Deferred@1371264324 after retry RPC after error@1253104221) -> (continuation of Deferred@1439256706 after retry RPC after error@964087912) -> (continuation of Deferred@912794615 after retry RPC after error@2059822421) -> (continuation of Deferred@1695836111 after retry RPC after error@1793990293), errback=(continuation of Deferred@504856616 after retry RPC after error@19714876) -> (continuation of Deferred@480226246 after retry RPC after error@1971183019) -> (continuation of Deferred@1472792044 after retry RPC after error@807115034) -> (continuation of Deferred@1371264324 after retry RPC after error@1253104221) -> (continuation of Deferred@1439256706 after retry RPC after error@964087912) -> (continuation of Deferred@912794615 after retry RPC after error@2059822421) -> (continuation of Deferred@1695836111 after retry RPC after error@1793990293)))
03:00:16.519 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.520 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.520 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.520 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.520 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.520 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.520 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.541 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.541 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.541 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.541 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.541 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.541 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.561 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.561 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.561 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.561 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.561 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.561 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.561 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.580 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.580 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.580 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.580 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.580 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.580 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.599 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.600 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.600 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.600 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.600 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.600 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.600 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.619 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.620 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.620 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.620 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.620 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.620 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.620 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.639 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.640 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.640 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.640 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.640 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.640 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.640 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.760 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.760 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.761 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.761 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.761 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.761 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.762 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1650364864(state=PENDING, result=null, callback=(continuation of Deferred@1760784223 after retry RPC after error@1173320107) -> (continuation of Deferred@2006945693 after retry RPC after error@1633271871) -> (continuation of Deferred@312726236 after retry RPC after error@746560266) -> (continuation of Deferred@655821439 after retry RPC after error@148926106) -> (continuation of Deferred@81361000 after retry RPC after error@1412301874) -> (continuation of Deferred@1274390586 after retry RPC after error@1089112928) -> (continuation of Deferred@264220973 after retry RPC after error@722746236), errback=(continuation of Deferred@1760784223 after retry RPC after error@1173320107) -> (continuation of Deferred@2006945693 after retry RPC after error@1633271871) -> (continuation of Deferred@312726236 after retry RPC after error@746560266) -> (continuation of Deferred@655821439 after retry RPC after error@148926106) -> (continuation of Deferred@81361000 after retry RPC after error@1412301874) -> (continuation of Deferred@1274390586 after retry RPC after error@1089112928) -> (continuation of Deferred@264220973 after retry RPC after error@722746236)))
03:00:16.762 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1650364864(state=PENDING, result=null, callback=(continuation of Deferred@1760784223 after retry RPC after error@1173320107) -> (continuation of Deferred@2006945693 after retry RPC after error@1633271871) -> (continuation of Deferred@312726236 after retry RPC after error@746560266) -> (continuation of Deferred@655821439 after retry RPC after error@148926106) -> (continuation of Deferred@81361000 after retry RPC after error@1412301874) -> (continuation of Deferred@1274390586 after retry RPC after error@1089112928) -> (continuation of Deferred@264220973 after retry RPC after error@722746236), errback=(continuation of Deferred@1760784223 after retry RPC after error@1173320107) -> (continuation of Deferred@2006945693 after retry RPC after error@1633271871) -> (continuation of Deferred@312726236 after retry RPC after error@746560266) -> (continuation of Deferred@655821439 after retry RPC after error@148926106) -> (continuation of Deferred@81361000 after retry RPC after error@1412301874) -> (continuation of Deferred@1274390586 after retry RPC after error@1089112928) -> (continuation of Deferred@264220973 after retry RPC after error@722746236)))
03:00:16.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.780 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.780 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.780 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.780 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.781 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.781 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.799 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.800 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.800 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.800 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.800 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.800 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.800 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.819 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.819 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.820 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.820 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.820 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.820 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.820 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.820 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.841 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.841 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.841 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.841 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.841 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.841 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.860 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.860 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.860 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.861 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.861 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.861 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.881 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.881 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.881 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.881 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.881 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.881 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.941 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.941 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.941 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.941 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.941 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.941 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.942 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.942 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [163ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1531770470(state=PENDING, result=null, callback=(continuation of Deferred@484458297 after retry RPC after error@242989167) -> (continuation of Deferred@1049231264 after retry RPC after error@38457809) -> (continuation of Deferred@525217622 after retry RPC after error@1277510672) -> (continuation of Deferred@1033946827 after retry RPC after error@1773161719) -> (continuation of Deferred@124447240 after retry RPC after error@1627994929) -> (continuation of Deferred@1512320837 after retry RPC after error@823658242), errback=(continuation of Deferred@484458297 after retry RPC after error@242989167) -> (continuation of Deferred@1049231264 after retry RPC after error@38457809) -> (continuation of Deferred@525217622 after retry RPC after error@1277510672) -> (continuation of Deferred@1033946827 after retry RPC after error@1773161719) -> (continuation of Deferred@124447240 after retry RPC after error@1627994929) -> (continuation of Deferred@1512320837 after retry RPC after error@823658242)))
03:00:16.943 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [163ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1531770470(state=PENDING, result=null, callback=(continuation of Deferred@484458297 after retry RPC after error@242989167) -> (continuation of Deferred@1049231264 after retry RPC after error@38457809) -> (continuation of Deferred@525217622 after retry RPC after error@1277510672) -> (continuation of Deferred@1033946827 after retry RPC after error@1773161719) -> (continuation of Deferred@124447240 after retry RPC after error@1627994929) -> (continuation of Deferred@1512320837 after retry RPC after error@823658242), errback=(continuation of Deferred@484458297 after retry RPC after error@242989167) -> (continuation of Deferred@1049231264 after retry RPC after error@38457809) -> (continuation of Deferred@525217622 after retry RPC after error@1277510672) -> (continuation of Deferred@1033946827 after retry RPC after error@1773161719) -> (continuation of Deferred@124447240 after retry RPC after error@1627994929) -> (continuation of Deferred@1512320837 after retry RPC after error@823658242)))
03:00:16.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.961 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.961 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.961 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.961 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.961 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.961 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:16.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:16.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:16.980 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.980 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.980 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.981 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:16.981 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.981 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:16.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.000 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.000 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.000 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.000 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.000 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.000 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.019 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.020 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.020 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.020 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.020 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.020 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.020 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.041 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.041 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.041 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.041 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.041 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.041 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.042 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.060 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.060 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.060 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.060 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.060 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.061 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.141 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.141 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.141 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.141 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.141 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.141 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.142 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.201 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.201 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.201 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.201 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.201 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.201 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.202 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@922087529(state=PENDING, result=null, callback=(continuation of Deferred@680532894 after retry RPC after error@1736580366) -> (continuation of Deferred@1345550365 after retry RPC after error@1239203018) -> (continuation of Deferred@1639449220 after retry RPC after error@379918800) -> (continuation of Deferred@1570597001 after retry RPC after error@1904981276) -> (continuation of Deferred@1342584203 after retry RPC after error@1859374530) -> (continuation of Deferred@1488726449 after retry RPC after error@1957909743) -> (continuation of Deferred@360226207 after retry RPC after error@1706426159), errback=(continuation of Deferred@680532894 after retry RPC after error@1736580366) -> (continuation of Deferred@1345550365 after retry RPC after error@1239203018) -> (continuation of Deferred@1639449220 after retry RPC after error@379918800) -> (continuation of Deferred@1570597001 after retry RPC after error@1904981276) -> (continuation of Deferred@1342584203 after retry RPC after error@1859374530) -> (continuation of Deferred@1488726449 after retry RPC after error@1957909743) -> (continuation of Deferred@360226207 after retry RPC after error@1706426159)))
03:00:17.202 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@922087529(state=PENDING, result=null, callback=(continuation of Deferred@680532894 after retry RPC after error@1736580366) -> (continuation of Deferred@1345550365 after retry RPC after error@1239203018) -> (continuation of Deferred@1639449220 after retry RPC after error@379918800) -> (continuation of Deferred@1570597001 after retry RPC after error@1904981276) -> (continuation of Deferred@1342584203 after retry RPC after error@1859374530) -> (continuation of Deferred@1488726449 after retry RPC after error@1957909743) -> (continuation of Deferred@360226207 after retry RPC after error@1706426159), errback=(continuation of Deferred@680532894 after retry RPC after error@1736580366) -> (continuation of Deferred@1345550365 after retry RPC after error@1239203018) -> (continuation of Deferred@1639449220 after retry RPC after error@379918800) -> (continuation of Deferred@1570597001 after retry RPC after error@1904981276) -> (continuation of Deferred@1342584203 after retry RPC after error@1859374530) -> (continuation of Deferred@1488726449 after retry RPC after error@1957909743) -> (continuation of Deferred@360226207 after retry RPC after error@1706426159)))
03:00:17.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.220 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.220 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.221 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.221 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.221 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.221 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.222 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.241 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.241 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.241 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.242 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.242 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.242 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.242 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.260 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.260 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.260 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.260 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.260 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.260 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.261 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.279 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.280 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.280 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.280 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.280 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.280 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.280 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.301 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.301 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.301 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.301 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.301 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.301 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.302 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.321 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.321 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.321 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.321 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.321 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.321 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.381 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.381 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.381 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.381 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.381 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.381 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.420 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.421 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.421 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.421 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.421 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.421 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.421 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.422 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@370902282(state=PENDING, result=null, callback=(continuation of Deferred@1712484777 after retry RPC after error@1793029148) -> (continuation of Deferred@1460209077 after retry RPC after error@502327691) -> (continuation of Deferred@1746558117 after retry RPC after error@2037688878) -> (continuation of Deferred@1716468760 after retry RPC after error@2095656427) -> (continuation of Deferred@1177336231 after retry RPC after error@441802390) -> (continuation of Deferred@90512952 after retry RPC after error@811003146) -> (continuation of Deferred@320117542 after retry RPC after error@595973692), errback=(continuation of Deferred@1712484777 after retry RPC after error@1793029148) -> (continuation of Deferred@1460209077 after retry RPC after error@502327691) -> (continuation of Deferred@1746558117 after retry RPC after error@2037688878) -> (continuation of Deferred@1716468760 after retry RPC after error@2095656427) -> (continuation of Deferred@1177336231 after retry RPC after error@441802390) -> (continuation of Deferred@90512952 after retry RPC after error@811003146) -> (continuation of Deferred@320117542 after retry RPC after error@595973692)))
03:00:17.422 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@370902282(state=PENDING, result=null, callback=(continuation of Deferred@1712484777 after retry RPC after error@1793029148) -> (continuation of Deferred@1460209077 after retry RPC after error@502327691) -> (continuation of Deferred@1746558117 after retry RPC after error@2037688878) -> (continuation of Deferred@1716468760 after retry RPC after error@2095656427) -> (continuation of Deferred@1177336231 after retry RPC after error@441802390) -> (continuation of Deferred@90512952 after retry RPC after error@811003146) -> (continuation of Deferred@320117542 after retry RPC after error@595973692), errback=(continuation of Deferred@1712484777 after retry RPC after error@1793029148) -> (continuation of Deferred@1460209077 after retry RPC after error@502327691) -> (continuation of Deferred@1746558117 after retry RPC after error@2037688878) -> (continuation of Deferred@1716468760 after retry RPC after error@2095656427) -> (continuation of Deferred@1177336231 after retry RPC after error@441802390) -> (continuation of Deferred@90512952 after retry RPC after error@811003146) -> (continuation of Deferred@320117542 after retry RPC after error@595973692)))
03:00:17.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.440 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.440 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.440 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.440 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.440 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.440 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.459 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.459 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.460 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.460 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.460 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.460 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.460 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.460 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.461 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.481 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.481 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.481 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.481 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.482 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.482 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.501 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.501 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.501 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.501 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.501 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.501 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.520 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.520 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.521 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.521 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.521 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.521 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.540 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.540 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.540 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.540 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.540 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.540 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.579 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.580 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.580 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.580 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.580 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.580 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.580 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.640 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.640 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.640 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.640 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.641 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.641 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.641 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [47ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [47ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2024234370(state=PENDING, result=null, callback=(continuation of Deferred@1031144228 after retry RPC after error@1039531968) -> (continuation of Deferred@1311657801 after retry RPC after error@203898947) -> (continuation of Deferred@29691328 after retry RPC after error@333244871) -> (continuation of Deferred@800368882 after retry RPC after error@430424095) -> (continuation of Deferred@816149990 after retry RPC after error@1145254116) -> (continuation of Deferred@1320874918 after retry RPC after error@1773823318) -> (continuation of Deferred@1923823554 after retry RPC after error@791870819), errback=(continuation of Deferred@1031144228 after retry RPC after error@1039531968) -> (continuation of Deferred@1311657801 after retry RPC after error@203898947) -> (continuation of Deferred@29691328 after retry RPC after error@333244871) -> (continuation of Deferred@800368882 after retry RPC after error@430424095) -> (continuation of Deferred@816149990 after retry RPC after error@1145254116) -> (continuation of Deferred@1320874918 after retry RPC after error@1773823318) -> (continuation of Deferred@1923823554 after retry RPC after error@791870819)))
03:00:17.642 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [47ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [47ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2024234370(state=PENDING, result=null, callback=(continuation of Deferred@1031144228 after retry RPC after error@1039531968) -> (continuation of Deferred@1311657801 after retry RPC after error@203898947) -> (continuation of Deferred@29691328 after retry RPC after error@333244871) -> (continuation of Deferred@800368882 after retry RPC after error@430424095) -> (continuation of Deferred@816149990 after retry RPC after error@1145254116) -> (continuation of Deferred@1320874918 after retry RPC after error@1773823318) -> (continuation of Deferred@1923823554 after retry RPC after error@791870819), errback=(continuation of Deferred@1031144228 after retry RPC after error@1039531968) -> (continuation of Deferred@1311657801 after retry RPC after error@203898947) -> (continuation of Deferred@29691328 after retry RPC after error@333244871) -> (continuation of Deferred@800368882 after retry RPC after error@430424095) -> (continuation of Deferred@816149990 after retry RPC after error@1145254116) -> (continuation of Deferred@1320874918 after retry RPC after error@1773823318) -> (continuation of Deferred@1923823554 after retry RPC after error@791870819)))
03:00:17.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.660 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.660 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.660 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.660 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.660 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.660 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.679 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.679 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.680 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.680 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.680 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.680 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.680 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.680 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.701 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.701 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.701 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.701 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.701 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.701 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.720 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.721 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.721 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.721 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.721 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.721 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.721 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.740 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.740 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.740 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.741 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.741 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.741 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.760 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.760 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.760 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.761 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.761 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.761 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.841 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.841 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.841 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.841 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.841 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.841 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.919 [DEBUG - pool-40-thread-1] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1267938445(state=PENDING, result=null, callback=(continuation of Deferred@809399454 after retry RPC after error@1988126715) -> (continuation of Deferred@1732231455 after retry RPC after error@903521833) -> (continuation of Deferred@1988341384 after retry RPC after error@55156764) -> (continuation of Deferred@451391059 after retry RPC after error@2053945544) -> (continuation of Deferred@300990179 after retry RPC after error@1610065281) -> (continuation of Deferred@2026183962 after retry RPC after error@1743798322) -> (continuation of Deferred@1944321417 after retry RPC after error@291149273), errback=(continuation of Deferred@809399454 after retry RPC after error@1988126715) -> (continuation of Deferred@1732231455 after retry RPC after error@903521833) -> (continuation of Deferred@1988341384 after retry RPC after error@55156764) -> (continuation of Deferred@451391059 after retry RPC after error@2053945544) -> (continuation of Deferred@300990179 after retry RPC after error@1610065281) -> (continuation of Deferred@2026183962 after retry RPC after error@1743798322) -> (continuation of Deferred@1944321417 after retry RPC after error@291149273)))
03:00:17.920 [WARN - pool-40-thread-1] (HashedWheelTimer.java:729) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
03:00:17.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.940 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.940 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.940 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.940 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.940 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.940 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.959 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.960 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.960 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.960 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.960 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.960 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.960 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:17.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:17.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:17.981 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.981 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.981 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.981 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:17.981 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.981 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:17.982 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.000 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.001 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.001 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.001 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.001 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.001 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.020 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.020 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.021 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.021 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.021 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.021 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.060 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.060 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.061 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.061 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.061 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.061 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.100 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.100 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.100 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.100 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.101 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.101 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.101 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1420801123(state=PENDING, result=null, callback=(continuation of Deferred@445906404 after retry RPC after error@1282106131) -> (continuation of Deferred@481778905 after retry RPC after error@890437504) -> (continuation of Deferred@812337179 after retry RPC after error@663738110) -> (continuation of Deferred@24647490 after retry RPC after error@599004690) -> (continuation of Deferred@1232452079 after retry RPC after error@269790745) -> (continuation of Deferred@1368877406 after retry RPC after error@1385226744), errback=(continuation of Deferred@445906404 after retry RPC after error@1282106131) -> (continuation of Deferred@481778905 after retry RPC after error@890437504) -> (continuation of Deferred@812337179 after retry RPC after error@663738110) -> (continuation of Deferred@24647490 after retry RPC after error@599004690) -> (continuation of Deferred@1232452079 after retry RPC after error@269790745) -> (continuation of Deferred@1368877406 after retry RPC after error@1385226744)))
03:00:18.102 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1420801123(state=PENDING, result=null, callback=(continuation of Deferred@445906404 after retry RPC after error@1282106131) -> (continuation of Deferred@481778905 after retry RPC after error@890437504) -> (continuation of Deferred@812337179 after retry RPC after error@663738110) -> (continuation of Deferred@24647490 after retry RPC after error@599004690) -> (continuation of Deferred@1232452079 after retry RPC after error@269790745) -> (continuation of Deferred@1368877406 after retry RPC after error@1385226744), errback=(continuation of Deferred@445906404 after retry RPC after error@1282106131) -> (continuation of Deferred@481778905 after retry RPC after error@890437504) -> (continuation of Deferred@812337179 after retry RPC after error@663738110) -> (continuation of Deferred@24647490 after retry RPC after error@599004690) -> (continuation of Deferred@1232452079 after retry RPC after error@269790745) -> (continuation of Deferred@1368877406 after retry RPC after error@1385226744)))
03:00:18.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.120 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.120 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.120 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.120 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.120 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.120 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.141 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.141 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.141 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.141 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.141 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.141 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.141 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.159 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.160 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.160 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.160 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.160 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.161 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.161 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.179 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.180 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.180 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.180 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.180 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.180 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.181 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.199 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.199 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.200 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.200 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.200 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.200 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.200 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.200 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.241 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.241 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.241 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.241 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.241 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.241 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.242 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.319 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.320 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.320 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.320 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.320 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.320 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.320 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.321 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1764278406(state=PENDING, result=null, callback=(continuation of Deferred@1013080954 after retry RPC after error@2129305282) -> (continuation of Deferred@19606441 after retry RPC after error@782825466) -> (continuation of Deferred@33433830 after retry RPC after error@726182549) -> (continuation of Deferred@1936112182 after retry RPC after error@446203015) -> (continuation of Deferred@1783707178 after retry RPC after error@69416439) -> (continuation of Deferred@1121097807 after retry RPC after error@472940590), errback=(continuation of Deferred@1013080954 after retry RPC after error@2129305282) -> (continuation of Deferred@19606441 after retry RPC after error@782825466) -> (continuation of Deferred@33433830 after retry RPC after error@726182549) -> (continuation of Deferred@1936112182 after retry RPC after error@446203015) -> (continuation of Deferred@1783707178 after retry RPC after error@69416439) -> (continuation of Deferred@1121097807 after retry RPC after error@472940590)))
03:00:18.321 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1764278406(state=PENDING, result=null, callback=(continuation of Deferred@1013080954 after retry RPC after error@2129305282) -> (continuation of Deferred@19606441 after retry RPC after error@782825466) -> (continuation of Deferred@33433830 after retry RPC after error@726182549) -> (continuation of Deferred@1936112182 after retry RPC after error@446203015) -> (continuation of Deferred@1783707178 after retry RPC after error@69416439) -> (continuation of Deferred@1121097807 after retry RPC after error@472940590), errback=(continuation of Deferred@1013080954 after retry RPC after error@2129305282) -> (continuation of Deferred@19606441 after retry RPC after error@782825466) -> (continuation of Deferred@33433830 after retry RPC after error@726182549) -> (continuation of Deferred@1936112182 after retry RPC after error@446203015) -> (continuation of Deferred@1783707178 after retry RPC after error@69416439) -> (continuation of Deferred@1121097807 after retry RPC after error@472940590)))
03:00:18.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.341 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.341 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.341 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.341 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.341 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.341 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.360 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.360 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.360 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.360 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.360 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.361 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.361 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.361 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.379 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.380 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.380 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.380 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.380 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.380 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.380 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.399 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.400 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.400 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.400 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.400 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.400 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.400 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.421 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.421 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.421 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.421 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.421 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.421 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.421 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.461 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.461 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.461 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.461 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.461 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.461 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.462 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.481 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.481 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.481 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.481 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.481 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.481 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.481 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.521 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.521 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.521 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.521 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.521 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.521 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.522 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.599 [DEBUG - pool-40-thread-1] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1828252917(state=PENDING, result=null, callback=(continuation of Deferred@1250901958 after retry RPC after error@1559780202) -> (continuation of Deferred@2020957404 after retry RPC after error@1405192291) -> (continuation of Deferred@1619262674 after retry RPC after error@2118047907) -> (continuation of Deferred@746579972 after retry RPC after error@1940062072) -> (continuation of Deferred@213448882 after retry RPC after error@72073798) -> (continuation of Deferred@1470575875 after retry RPC after error@865749424) -> (continuation of Deferred@2016168473 after retry RPC after error@573566632) -> (continuation of Deferred@1288620227 after retry RPC after error@762892067), errback=(continuation of Deferred@1250901958 after retry RPC after error@1559780202) -> (continuation of Deferred@2020957404 after retry RPC after error@1405192291) -> (continuation of Deferred@1619262674 after retry RPC after error@2118047907) -> (continuation of Deferred@746579972 after retry RPC after error@1940062072) -> (continuation of Deferred@213448882 after retry RPC after error@72073798) -> (continuation of Deferred@1470575875 after retry RPC after error@865749424) -> (continuation of Deferred@2016168473 after retry RPC after error@573566632) -> (continuation of Deferred@1288620227 after retry RPC after error@762892067)))
03:00:18.600 [WARN - pool-40-thread-1] (HashedWheelTimer.java:729) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
03:00:18.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.620 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.620 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.620 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.620 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.620 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.620 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.639 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.640 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.640 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.640 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.640 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.640 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.640 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.641 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.641 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.641 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.641 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.641 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.641 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.641 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.661 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.661 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.661 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.661 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.661 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.661 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.662 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.681 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.681 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.681 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.681 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.681 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.681 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.700 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.700 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.700 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.700 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.700 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.700 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.740 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.740 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.740 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.740 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.740 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.740 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.759 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.760 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.760 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.760 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.760 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.760 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.760 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.860 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.860 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.861 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.861 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.861 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.861 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.862 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1582748987(state=PENDING, result=null, callback=(continuation of Deferred@1921736224 after retry RPC after error@1763028889) -> (continuation of Deferred@411197596 after retry RPC after error@1880461754) -> (continuation of Deferred@1481096134 after retry RPC after error@485146160) -> (continuation of Deferred@392670129 after retry RPC after error@1260609702) -> (continuation of Deferred@786398095 after retry RPC after error@1728928602) -> (continuation of Deferred@1302158889 after retry RPC after error@501948780) -> (continuation of Deferred@1211966045 after retry RPC after error@1164404619), errback=(continuation of Deferred@1921736224 after retry RPC after error@1763028889) -> (continuation of Deferred@411197596 after retry RPC after error@1880461754) -> (continuation of Deferred@1481096134 after retry RPC after error@485146160) -> (continuation of Deferred@392670129 after retry RPC after error@1260609702) -> (continuation of Deferred@786398095 after retry RPC after error@1728928602) -> (continuation of Deferred@1302158889 after retry RPC after error@501948780) -> (continuation of Deferred@1211966045 after retry RPC after error@1164404619)))
03:00:18.862 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1582748987(state=PENDING, result=null, callback=(continuation of Deferred@1921736224 after retry RPC after error@1763028889) -> (continuation of Deferred@411197596 after retry RPC after error@1880461754) -> (continuation of Deferred@1481096134 after retry RPC after error@485146160) -> (continuation of Deferred@392670129 after retry RPC after error@1260609702) -> (continuation of Deferred@786398095 after retry RPC after error@1728928602) -> (continuation of Deferred@1302158889 after retry RPC after error@501948780) -> (continuation of Deferred@1211966045 after retry RPC after error@1164404619), errback=(continuation of Deferred@1921736224 after retry RPC after error@1763028889) -> (continuation of Deferred@411197596 after retry RPC after error@1880461754) -> (continuation of Deferred@1481096134 after retry RPC after error@485146160) -> (continuation of Deferred@392670129 after retry RPC after error@1260609702) -> (continuation of Deferred@786398095 after retry RPC after error@1728928602) -> (continuation of Deferred@1302158889 after retry RPC after error@501948780) -> (continuation of Deferred@1211966045 after retry RPC after error@1164404619)))
03:00:18.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.880 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.880 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.880 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.880 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.880 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.880 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.899 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.900 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.900 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.900 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.900 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.900 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.900 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.919 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.919 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.920 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.920 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.920 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.920 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.920 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.920 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.941 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.941 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.941 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.941 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.942 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.942 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.942 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.960 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.961 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.961 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.961 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.961 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.961 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:18.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:18.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:18.980 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.980 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.980 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:18.980 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.980 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.980 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:18.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.019 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.020 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.020 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.020 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.020 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.020 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.020 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.021 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@117856566(state=PENDING, result=null, callback=(continuation of Deferred@2032348514 after retry RPC after error@2074469259) -> (continuation of Deferred@403509047 after retry RPC after error@762788585) -> (continuation of Deferred@1306479128 after retry RPC after error@422728083) -> (continuation of Deferred@1773568309 after retry RPC after error@2105695896) -> (continuation of Deferred@1245332666 after retry RPC after error@1099901607) -> (continuation of Deferred@485336202 after retry RPC after error@1937002257), errback=(continuation of Deferred@2032348514 after retry RPC after error@2074469259) -> (continuation of Deferred@403509047 after retry RPC after error@762788585) -> (continuation of Deferred@1306479128 after retry RPC after error@422728083) -> (continuation of Deferred@1773568309 after retry RPC after error@2105695896) -> (continuation of Deferred@1245332666 after retry RPC after error@1099901607) -> (continuation of Deferred@485336202 after retry RPC after error@1937002257)))
03:00:19.021 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@117856566(state=PENDING, result=null, callback=(continuation of Deferred@2032348514 after retry RPC after error@2074469259) -> (continuation of Deferred@403509047 after retry RPC after error@762788585) -> (continuation of Deferred@1306479128 after retry RPC after error@422728083) -> (continuation of Deferred@1773568309 after retry RPC after error@2105695896) -> (continuation of Deferred@1245332666 after retry RPC after error@1099901607) -> (continuation of Deferred@485336202 after retry RPC after error@1937002257), errback=(continuation of Deferred@2032348514 after retry RPC after error@2074469259) -> (continuation of Deferred@403509047 after retry RPC after error@762788585) -> (continuation of Deferred@1306479128 after retry RPC after error@422728083) -> (continuation of Deferred@1773568309 after retry RPC after error@2105695896) -> (continuation of Deferred@1245332666 after retry RPC after error@1099901607) -> (continuation of Deferred@485336202 after retry RPC after error@1937002257)))
03:00:19.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.041 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.041 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.041 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.041 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.041 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.041 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.042 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.060 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.060 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.061 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.061 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.061 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.061 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.080 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.080 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.080 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.080 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.080 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.081 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.081 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.099 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.100 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.100 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.100 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.100 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.100 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.100 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.119 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.119 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.120 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.120 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.120 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.120 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.120 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.120 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.161 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.161 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.161 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.161 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.161 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.161 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.181 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.181 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.181 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.181 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.181 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.181 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.279 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.279 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.280 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.280 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.281 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.281 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.281 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.281 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.281 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@102763112(state=PENDING, result=null, callback=(continuation of Deferred@1570125945 after retry RPC after error@915457749) -> (continuation of Deferred@1644741453 after retry RPC after error@1685976622) -> (continuation of Deferred@138698536 after retry RPC after error@953325211) -> (continuation of Deferred@484714599 after retry RPC after error@396583251) -> (continuation of Deferred@1026871696 after retry RPC after error@301919971) -> (continuation of Deferred@744774571 after retry RPC after error@541134477) -> (continuation of Deferred@1004729156 after retry RPC after error@1569561389), errback=(continuation of Deferred@1570125945 after retry RPC after error@915457749) -> (continuation of Deferred@1644741453 after retry RPC after error@1685976622) -> (continuation of Deferred@138698536 after retry RPC after error@953325211) -> (continuation of Deferred@484714599 after retry RPC after error@396583251) -> (continuation of Deferred@1026871696 after retry RPC after error@301919971) -> (continuation of Deferred@744774571 after retry RPC after error@541134477) -> (continuation of Deferred@1004729156 after retry RPC after error@1569561389)))
03:00:19.282 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@102763112(state=PENDING, result=null, callback=(continuation of Deferred@1570125945 after retry RPC after error@915457749) -> (continuation of Deferred@1644741453 after retry RPC after error@1685976622) -> (continuation of Deferred@138698536 after retry RPC after error@953325211) -> (continuation of Deferred@484714599 after retry RPC after error@396583251) -> (continuation of Deferred@1026871696 after retry RPC after error@301919971) -> (continuation of Deferred@744774571 after retry RPC after error@541134477) -> (continuation of Deferred@1004729156 after retry RPC after error@1569561389), errback=(continuation of Deferred@1570125945 after retry RPC after error@915457749) -> (continuation of Deferred@1644741453 after retry RPC after error@1685976622) -> (continuation of Deferred@138698536 after retry RPC after error@953325211) -> (continuation of Deferred@484714599 after retry RPC after error@396583251) -> (continuation of Deferred@1026871696 after retry RPC after error@301919971) -> (continuation of Deferred@744774571 after retry RPC after error@541134477) -> (continuation of Deferred@1004729156 after retry RPC after error@1569561389)))
03:00:19.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.301 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.301 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.301 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.301 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.301 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.301 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.302 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.320 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.320 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.320 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.320 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.321 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.321 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.322 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.340 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.340 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.340 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.340 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.340 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.340 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.359 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.360 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.360 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.360 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.360 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.360 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.360 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.379 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.379 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.380 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.380 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.380 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.380 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.380 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.380 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.400 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.401 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.401 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.401 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.401 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.401 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.402 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.461 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.461 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.461 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.461 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.461 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.461 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.462 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.499 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.500 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.500 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.500 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.500 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.500 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.500 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.501 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1694467863(state=PENDING, result=null, callback=(continuation of Deferred@604312195 after retry RPC after error@1837449287) -> (continuation of Deferred@940911278 after retry RPC after error@550163844) -> (continuation of Deferred@743918100 after retry RPC after error@1796714657) -> (continuation of Deferred@2059843383 after retry RPC after error@215362334) -> (continuation of Deferred@440911738 after retry RPC after error@1283718699) -> (continuation of Deferred@62101379 after retry RPC after error@1048629808) -> (continuation of Deferred@442023565 after retry RPC after error@1241086283), errback=(continuation of Deferred@604312195 after retry RPC after error@1837449287) -> (continuation of Deferred@940911278 after retry RPC after error@550163844) -> (continuation of Deferred@743918100 after retry RPC after error@1796714657) -> (continuation of Deferred@2059843383 after retry RPC after error@215362334) -> (continuation of Deferred@440911738 after retry RPC after error@1283718699) -> (continuation of Deferred@62101379 after retry RPC after error@1048629808) -> (continuation of Deferred@442023565 after retry RPC after error@1241086283)))
03:00:19.501 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1694467863(state=PENDING, result=null, callback=(continuation of Deferred@604312195 after retry RPC after error@1837449287) -> (continuation of Deferred@940911278 after retry RPC after error@550163844) -> (continuation of Deferred@743918100 after retry RPC after error@1796714657) -> (continuation of Deferred@2059843383 after retry RPC after error@215362334) -> (continuation of Deferred@440911738 after retry RPC after error@1283718699) -> (continuation of Deferred@62101379 after retry RPC after error@1048629808) -> (continuation of Deferred@442023565 after retry RPC after error@1241086283), errback=(continuation of Deferred@604312195 after retry RPC after error@1837449287) -> (continuation of Deferred@940911278 after retry RPC after error@550163844) -> (continuation of Deferred@743918100 after retry RPC after error@1796714657) -> (continuation of Deferred@2059843383 after retry RPC after error@215362334) -> (continuation of Deferred@440911738 after retry RPC after error@1283718699) -> (continuation of Deferred@62101379 after retry RPC after error@1048629808) -> (continuation of Deferred@442023565 after retry RPC after error@1241086283)))
03:00:19.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.521 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.521 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.521 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.521 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.521 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.521 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.522 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.541 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.541 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.541 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.541 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.541 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.541 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.541 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.542 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.561 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.561 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.561 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.561 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.561 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.561 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.562 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.580 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.580 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.581 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.581 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.581 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.581 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.599 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.600 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.600 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.600 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.600 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.600 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.600 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.639 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.640 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.640 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.640 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.640 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.640 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.640 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.699 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.700 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.700 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.700 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.700 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.700 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.700 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.702 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@720475696(state=PENDING, result=null, callback=(continuation of Deferred@1100221480 after retry RPC after error@1853537200) -> (continuation of Deferred@1267564269 after retry RPC after error@1844130234) -> (continuation of Deferred@209687907 after retry RPC after error@929241191) -> (continuation of Deferred@398029424 after retry RPC after error@594274268) -> (continuation of Deferred@1406122591 after retry RPC after error@2084581473) -> (continuation of Deferred@540469099 after retry RPC after error@1545580982), errback=(continuation of Deferred@1100221480 after retry RPC after error@1853537200) -> (continuation of Deferred@1267564269 after retry RPC after error@1844130234) -> (continuation of Deferred@209687907 after retry RPC after error@929241191) -> (continuation of Deferred@398029424 after retry RPC after error@594274268) -> (continuation of Deferred@1406122591 after retry RPC after error@2084581473) -> (continuation of Deferred@540469099 after retry RPC after error@1545580982)))
03:00:19.702 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@720475696(state=PENDING, result=null, callback=(continuation of Deferred@1100221480 after retry RPC after error@1853537200) -> (continuation of Deferred@1267564269 after retry RPC after error@1844130234) -> (continuation of Deferred@209687907 after retry RPC after error@929241191) -> (continuation of Deferred@398029424 after retry RPC after error@594274268) -> (continuation of Deferred@1406122591 after retry RPC after error@2084581473) -> (continuation of Deferred@540469099 after retry RPC after error@1545580982), errback=(continuation of Deferred@1100221480 after retry RPC after error@1853537200) -> (continuation of Deferred@1267564269 after retry RPC after error@1844130234) -> (continuation of Deferred@209687907 after retry RPC after error@929241191) -> (continuation of Deferred@398029424 after retry RPC after error@594274268) -> (continuation of Deferred@1406122591 after retry RPC after error@2084581473) -> (continuation of Deferred@540469099 after retry RPC after error@1545580982)))
03:00:19.719 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.719 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.720 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.720 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.720 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.720 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.721 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.721 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.721 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.741 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.741 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.741 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.741 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.741 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.741 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.759 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.760 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.760 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.760 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.760 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.760 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.760 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.779 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.779 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.780 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.780 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.780 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.780 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.780 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.780 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.800 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.800 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.800 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.801 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.801 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.801 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.840 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.840 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.840 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.840 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.840 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.841 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.900 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.900 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.900 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.901 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.901 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.901 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.919 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.920 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.920 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.920 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.920 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.920 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.920 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.921 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@741049358(state=PENDING, result=null, callback=(continuation of Deferred@866591247 after retry RPC after error@1852446554) -> (continuation of Deferred@246169392 after retry RPC after error@2128072991) -> (continuation of Deferred@1961501584 after retry RPC after error@1241791120) -> (continuation of Deferred@633264479 after retry RPC after error@1705098206) -> (continuation of Deferred@1081274969 after retry RPC after error@734625917) -> (continuation of Deferred@2028912762 after retry RPC after error@276425714) -> (continuation of Deferred@1204695966 after retry RPC after error@551786866), errback=(continuation of Deferred@866591247 after retry RPC after error@1852446554) -> (continuation of Deferred@246169392 after retry RPC after error@2128072991) -> (continuation of Deferred@1961501584 after retry RPC after error@1241791120) -> (continuation of Deferred@633264479 after retry RPC after error@1705098206) -> (continuation of Deferred@1081274969 after retry RPC after error@734625917) -> (continuation of Deferred@2028912762 after retry RPC after error@276425714) -> (continuation of Deferred@1204695966 after retry RPC after error@551786866)))
03:00:19.922 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@741049358(state=PENDING, result=null, callback=(continuation of Deferred@866591247 after retry RPC after error@1852446554) -> (continuation of Deferred@246169392 after retry RPC after error@2128072991) -> (continuation of Deferred@1961501584 after retry RPC after error@1241791120) -> (continuation of Deferred@633264479 after retry RPC after error@1705098206) -> (continuation of Deferred@1081274969 after retry RPC after error@734625917) -> (continuation of Deferred@2028912762 after retry RPC after error@276425714) -> (continuation of Deferred@1204695966 after retry RPC after error@551786866), errback=(continuation of Deferred@866591247 after retry RPC after error@1852446554) -> (continuation of Deferred@246169392 after retry RPC after error@2128072991) -> (continuation of Deferred@1961501584 after retry RPC after error@1241791120) -> (continuation of Deferred@633264479 after retry RPC after error@1705098206) -> (continuation of Deferred@1081274969 after retry RPC after error@734625917) -> (continuation of Deferred@2028912762 after retry RPC after error@276425714) -> (continuation of Deferred@1204695966 after retry RPC after error@551786866)))
03:00:19.939 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.939 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.940 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.940 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.940 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.940 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.940 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.940 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.960 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.960 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.960 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.961 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.961 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.961 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.979 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:19.980 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.980 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.980 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.980 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:19.980 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.980 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:19.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:19.999 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:19.999 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.000 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.000 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.000 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.000 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.000 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.000 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.020 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.021 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.021 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.021 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.021 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.021 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.060 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.060 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.060 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.060 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.060 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.060 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.100 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.100 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.100 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.100 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.100 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.100 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.102 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1453638034(state=PENDING, result=null, callback=(continuation of Deferred@1856450779 after retry RPC after error@1151977215) -> (continuation of Deferred@1944229115 after retry RPC after error@54158148) -> (continuation of Deferred@1414310416 after retry RPC after error@1591214168) -> (continuation of Deferred@972685876 after retry RPC after error@1762289164) -> (continuation of Deferred@1478970647 after retry RPC after error@1180912779) -> (continuation of Deferred@794889681 after retry RPC after error@367076407), errback=(continuation of Deferred@1856450779 after retry RPC after error@1151977215) -> (continuation of Deferred@1944229115 after retry RPC after error@54158148) -> (continuation of Deferred@1414310416 after retry RPC after error@1591214168) -> (continuation of Deferred@972685876 after retry RPC after error@1762289164) -> (continuation of Deferred@1478970647 after retry RPC after error@1180912779) -> (continuation of Deferred@794889681 after retry RPC after error@367076407)))
03:00:20.102 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1453638034(state=PENDING, result=null, callback=(continuation of Deferred@1856450779 after retry RPC after error@1151977215) -> (continuation of Deferred@1944229115 after retry RPC after error@54158148) -> (continuation of Deferred@1414310416 after retry RPC after error@1591214168) -> (continuation of Deferred@972685876 after retry RPC after error@1762289164) -> (continuation of Deferred@1478970647 after retry RPC after error@1180912779) -> (continuation of Deferred@794889681 after retry RPC after error@367076407), errback=(continuation of Deferred@1856450779 after retry RPC after error@1151977215) -> (continuation of Deferred@1944229115 after retry RPC after error@54158148) -> (continuation of Deferred@1414310416 after retry RPC after error@1591214168) -> (continuation of Deferred@972685876 after retry RPC after error@1762289164) -> (continuation of Deferred@1478970647 after retry RPC after error@1180912779) -> (continuation of Deferred@794889681 after retry RPC after error@367076407)))
03:00:20.119 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.120 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.120 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.120 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.120 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.120 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.120 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.141 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.141 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.141 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.141 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.141 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.141 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.142 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.161 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.161 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.161 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.161 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.161 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.161 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.180 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.180 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.180 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.181 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.181 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.181 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.200 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.200 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.200 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.200 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.200 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.200 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.239 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.240 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.240 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.240 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.240 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.240 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.241 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.241 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.299 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.300 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.300 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.300 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.300 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.300 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.300 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.302 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@56568484(state=PENDING, result=null, callback=(continuation of Deferred@349686969 after retry RPC after error@1626166860) -> (continuation of Deferred@1780243135 after retry RPC after error@275535698) -> (continuation of Deferred@989127650 after retry RPC after error@815893958) -> (continuation of Deferred@954261711 after retry RPC after error@1246029382) -> (continuation of Deferred@1462953759 after retry RPC after error@1807851394) -> (continuation of Deferred@1419308205 after retry RPC after error@956520110), errback=(continuation of Deferred@349686969 after retry RPC after error@1626166860) -> (continuation of Deferred@1780243135 after retry RPC after error@275535698) -> (continuation of Deferred@989127650 after retry RPC after error@815893958) -> (continuation of Deferred@954261711 after retry RPC after error@1246029382) -> (continuation of Deferred@1462953759 after retry RPC after error@1807851394) -> (continuation of Deferred@1419308205 after retry RPC after error@956520110)))
03:00:20.302 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@56568484(state=PENDING, result=null, callback=(continuation of Deferred@349686969 after retry RPC after error@1626166860) -> (continuation of Deferred@1780243135 after retry RPC after error@275535698) -> (continuation of Deferred@989127650 after retry RPC after error@815893958) -> (continuation of Deferred@954261711 after retry RPC after error@1246029382) -> (continuation of Deferred@1462953759 after retry RPC after error@1807851394) -> (continuation of Deferred@1419308205 after retry RPC after error@956520110), errback=(continuation of Deferred@349686969 after retry RPC after error@1626166860) -> (continuation of Deferred@1780243135 after retry RPC after error@275535698) -> (continuation of Deferred@989127650 after retry RPC after error@815893958) -> (continuation of Deferred@954261711 after retry RPC after error@1246029382) -> (continuation of Deferred@1462953759 after retry RPC after error@1807851394) -> (continuation of Deferred@1419308205 after retry RPC after error@956520110)))
03:00:20.319 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.319 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.320 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.320 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.320 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.320 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.320 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.320 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.341 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.341 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.341 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.341 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.341 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.341 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.360 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.360 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.360 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.360 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.360 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.361 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.379 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.380 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.380 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.380 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.381 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.381 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.381 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.399 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.399 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.400 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.400 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.400 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.400 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.400 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.400 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.441 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.441 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.441 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.441 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.441 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.441 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.460 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.460 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.461 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.461 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.461 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.461 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.462 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.501 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.501 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.501 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.501 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.501 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.501 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.502 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.561 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.561 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.561 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.561 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.561 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.561 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.562 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.562 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@476654132(state=PENDING, result=null, callback=(continuation of Deferred@2104477043 after retry RPC after error@326763402) -> (continuation of Deferred@941035810 after retry RPC after error@1836754078) -> (continuation of Deferred@1849536543 after retry RPC after error@756582401) -> (continuation of Deferred@755372673 after retry RPC after error@367005163) -> (continuation of Deferred@383711744 after retry RPC after error@2137993306) -> (continuation of Deferred@1650505516 after retry RPC after error@2012901854) -> (continuation of Deferred@378903515 after retry RPC after error@276726963) -> (continuation of Deferred@2000721991 after retry RPC after error@1393583088), errback=(continuation of Deferred@2104477043 after retry RPC after error@326763402) -> (continuation of Deferred@941035810 after retry RPC after error@1836754078) -> (continuation of Deferred@1849536543 after retry RPC after error@756582401) -> (continuation of Deferred@755372673 after retry RPC after error@367005163) -> (continuation of Deferred@383711744 after retry RPC after error@2137993306) -> (continuation of Deferred@1650505516 after retry RPC after error@2012901854) -> (continuation of Deferred@378903515 after retry RPC after error@276726963) -> (continuation of Deferred@2000721991 after retry RPC after error@1393583088)))
03:00:20.562 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@476654132(state=PENDING, result=null, callback=(continuation of Deferred@2104477043 after retry RPC after error@326763402) -> (continuation of Deferred@941035810 after retry RPC after error@1836754078) -> (continuation of Deferred@1849536543 after retry RPC after error@756582401) -> (continuation of Deferred@755372673 after retry RPC after error@367005163) -> (continuation of Deferred@383711744 after retry RPC after error@2137993306) -> (continuation of Deferred@1650505516 after retry RPC after error@2012901854) -> (continuation of Deferred@378903515 after retry RPC after error@276726963) -> (continuation of Deferred@2000721991 after retry RPC after error@1393583088), errback=(continuation of Deferred@2104477043 after retry RPC after error@326763402) -> (continuation of Deferred@941035810 after retry RPC after error@1836754078) -> (continuation of Deferred@1849536543 after retry RPC after error@756582401) -> (continuation of Deferred@755372673 after retry RPC after error@367005163) -> (continuation of Deferred@383711744 after retry RPC after error@2137993306) -> (continuation of Deferred@1650505516 after retry RPC after error@2012901854) -> (continuation of Deferred@378903515 after retry RPC after error@276726963) -> (continuation of Deferred@2000721991 after retry RPC after error@1393583088)))
03:00:20.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.580 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.580 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.580 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.580 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.580 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.580 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.599 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.600 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.600 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.600 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.600 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.600 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.600 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.621 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.621 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.621 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.621 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.621 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.621 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.622 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.641 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.641 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.641 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.641 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.641 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.641 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.641 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.641 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.642 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.642 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.660 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.660 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.660 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.660 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.660 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.661 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.679 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.680 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.680 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.680 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.680 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.680 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.680 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.719 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.720 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.720 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.720 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.720 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.720 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.720 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.721 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.779 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.780 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.780 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.780 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.780 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.780 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.780 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.839 [DEBUG - pool-40-thread-1] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [0ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@2092879188(state=PENDING, result=null, callback=(continuation of Deferred@660643133 after retry RPC after error@48983861) -> (continuation of Deferred@1790093768 after retry RPC after error@1626086823) -> (continuation of Deferred@1137114610 after retry RPC after error@437100664) -> (continuation of Deferred@1695462936 after retry RPC after error@1703906875) -> (continuation of Deferred@299025941 after retry RPC after error@1722526263) -> (continuation of Deferred@1988456076 after retry RPC after error@1868748849) -> (continuation of Deferred@1745308070 after retry RPC after error@1405529219) -> (continuation of Deferred@250239353 after retry RPC after error@2008071109), errback=(continuation of Deferred@660643133 after retry RPC after error@48983861) -> (continuation of Deferred@1790093768 after retry RPC after error@1626086823) -> (continuation of Deferred@1137114610 after retry RPC after error@437100664) -> (continuation of Deferred@1695462936 after retry RPC after error@1703906875) -> (continuation of Deferred@299025941 after retry RPC after error@1722526263) -> (continuation of Deferred@1988456076 after retry RPC after error@1868748849) -> (continuation of Deferred@1745308070 after retry RPC after error@1405529219) -> (continuation of Deferred@250239353 after retry RPC after error@2008071109)))
03:00:20.840 [WARN - pool-40-thread-1] (HashedWheelTimer.java:729) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
03:00:20.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.860 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.860 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.860 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.860 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.860 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.861 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.879 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.879 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.880 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.880 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.880 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.880 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.880 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.880 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.901 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.901 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.901 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.901 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.901 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.901 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.920 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.920 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.920 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.921 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.921 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.921 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.940 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.940 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.940 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.940 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.940 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.940 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:20.959 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:20.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:20.960 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.960 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.960 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.960 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.960 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:20.960 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:20.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.019 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.020 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.021 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.021 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.021 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.021 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.021 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.099 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.100 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.100 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.100 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.100 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.100 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.100 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.101 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@932606621(state=PENDING, result=null, callback=(continuation of Deferred@229690652 after retry RPC after error@1345197156) -> (continuation of Deferred@1434313025 after retry RPC after error@1375429609) -> (continuation of Deferred@1512611047 after retry RPC after error@2055548658) -> (continuation of Deferred@1086897930 after retry RPC after error@1252423889) -> (continuation of Deferred@1417087686 after retry RPC after error@131488772) -> (continuation of Deferred@48468763 after retry RPC after error@2071597926) -> (continuation of Deferred@389482485 after retry RPC after error@573853547), errback=(continuation of Deferred@229690652 after retry RPC after error@1345197156) -> (continuation of Deferred@1434313025 after retry RPC after error@1375429609) -> (continuation of Deferred@1512611047 after retry RPC after error@2055548658) -> (continuation of Deferred@1086897930 after retry RPC after error@1252423889) -> (continuation of Deferred@1417087686 after retry RPC after error@131488772) -> (continuation of Deferred@48468763 after retry RPC after error@2071597926) -> (continuation of Deferred@389482485 after retry RPC after error@573853547)))
03:00:21.102 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@932606621(state=PENDING, result=null, callback=(continuation of Deferred@229690652 after retry RPC after error@1345197156) -> (continuation of Deferred@1434313025 after retry RPC after error@1375429609) -> (continuation of Deferred@1512611047 after retry RPC after error@2055548658) -> (continuation of Deferred@1086897930 after retry RPC after error@1252423889) -> (continuation of Deferred@1417087686 after retry RPC after error@131488772) -> (continuation of Deferred@48468763 after retry RPC after error@2071597926) -> (continuation of Deferred@389482485 after retry RPC after error@573853547), errback=(continuation of Deferred@229690652 after retry RPC after error@1345197156) -> (continuation of Deferred@1434313025 after retry RPC after error@1375429609) -> (continuation of Deferred@1512611047 after retry RPC after error@2055548658) -> (continuation of Deferred@1086897930 after retry RPC after error@1252423889) -> (continuation of Deferred@1417087686 after retry RPC after error@131488772) -> (continuation of Deferred@48468763 after retry RPC after error@2071597926) -> (continuation of Deferred@389482485 after retry RPC after error@573853547)))
03:00:21.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.121 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.121 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.121 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.121 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.121 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.121 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.122 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.141 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.141 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.141 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.141 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.141 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.141 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.142 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.161 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.161 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.161 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.161 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.161 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.161 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.161 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.162 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.181 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.181 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.181 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.181 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.181 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.181 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.200 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.200 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.201 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.201 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.201 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.201 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.220 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.220 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.220 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.220 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.220 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.220 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.259 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.260 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.260 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.260 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.260 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.260 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.260 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.261 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.360 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.360 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.360 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.360 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.360 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.361 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.362 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@500606080(state=PENDING, result=null, callback=(continuation of Deferred@1067754382 after retry RPC after error@1187472416) -> (continuation of Deferred@1474788926 after retry RPC after error@2134717158) -> (continuation of Deferred@1704937052 after retry RPC after error@402970795) -> (continuation of Deferred@1962824475 after retry RPC after error@1648555360) -> (continuation of Deferred@2112942802 after retry RPC after error@1133493319) -> (continuation of Deferred@1164921254 after retry RPC after error@821255582) -> (continuation of Deferred@1248812848 after retry RPC after error@1958522814), errback=(continuation of Deferred@1067754382 after retry RPC after error@1187472416) -> (continuation of Deferred@1474788926 after retry RPC after error@2134717158) -> (continuation of Deferred@1704937052 after retry RPC after error@402970795) -> (continuation of Deferred@1962824475 after retry RPC after error@1648555360) -> (continuation of Deferred@2112942802 after retry RPC after error@1133493319) -> (continuation of Deferred@1164921254 after retry RPC after error@821255582) -> (continuation of Deferred@1248812848 after retry RPC after error@1958522814)))
03:00:21.362 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@500606080(state=PENDING, result=null, callback=(continuation of Deferred@1067754382 after retry RPC after error@1187472416) -> (continuation of Deferred@1474788926 after retry RPC after error@2134717158) -> (continuation of Deferred@1704937052 after retry RPC after error@402970795) -> (continuation of Deferred@1962824475 after retry RPC after error@1648555360) -> (continuation of Deferred@2112942802 after retry RPC after error@1133493319) -> (continuation of Deferred@1164921254 after retry RPC after error@821255582) -> (continuation of Deferred@1248812848 after retry RPC after error@1958522814), errback=(continuation of Deferred@1067754382 after retry RPC after error@1187472416) -> (continuation of Deferred@1474788926 after retry RPC after error@2134717158) -> (continuation of Deferred@1704937052 after retry RPC after error@402970795) -> (continuation of Deferred@1962824475 after retry RPC after error@1648555360) -> (continuation of Deferred@2112942802 after retry RPC after error@1133493319) -> (continuation of Deferred@1164921254 after retry RPC after error@821255582) -> (continuation of Deferred@1248812848 after retry RPC after error@1958522814)))
03:00:21.379 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.380 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.380 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.380 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.380 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.380 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.380 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.399 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.399 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.400 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.400 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.400 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.400 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.400 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.400 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.421 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.421 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.421 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.421 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.421 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.421 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.422 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.440 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.440 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.441 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.441 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.441 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.441 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.460 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.460 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.460 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.460 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.460 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.460 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.461 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.479 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.480 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.480 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.480 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.480 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.480 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.480 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.481 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.560 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.560 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.560 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.560 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.560 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.561 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.561 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.579 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.580 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.580 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.580 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.580 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.580 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.580 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.581 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@72393739(state=PENDING, result=null, callback=(continuation of Deferred@1191863545 after retry RPC after error@106568080) -> (continuation of Deferred@1391011512 after retry RPC after error@1402515474) -> (continuation of Deferred@1193356084 after retry RPC after error@291581145) -> (continuation of Deferred@170263723 after retry RPC after error@472399325) -> (continuation of Deferred@1377750503 after retry RPC after error@1249977497) -> (continuation of Deferred@1891131966 after retry RPC after error@757296929) -> (continuation of Deferred@167181975 after retry RPC after error@1250997980), errback=(continuation of Deferred@1191863545 after retry RPC after error@106568080) -> (continuation of Deferred@1391011512 after retry RPC after error@1402515474) -> (continuation of Deferred@1193356084 after retry RPC after error@291581145) -> (continuation of Deferred@170263723 after retry RPC after error@472399325) -> (continuation of Deferred@1377750503 after retry RPC after error@1249977497) -> (continuation of Deferred@1891131966 after retry RPC after error@757296929) -> (continuation of Deferred@167181975 after retry RPC after error@1250997980)))
03:00:21.581 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@72393739(state=PENDING, result=null, callback=(continuation of Deferred@1191863545 after retry RPC after error@106568080) -> (continuation of Deferred@1391011512 after retry RPC after error@1402515474) -> (continuation of Deferred@1193356084 after retry RPC after error@291581145) -> (continuation of Deferred@170263723 after retry RPC after error@472399325) -> (continuation of Deferred@1377750503 after retry RPC after error@1249977497) -> (continuation of Deferred@1891131966 after retry RPC after error@757296929) -> (continuation of Deferred@167181975 after retry RPC after error@1250997980), errback=(continuation of Deferred@1191863545 after retry RPC after error@106568080) -> (continuation of Deferred@1391011512 after retry RPC after error@1402515474) -> (continuation of Deferred@1193356084 after retry RPC after error@291581145) -> (continuation of Deferred@170263723 after retry RPC after error@472399325) -> (continuation of Deferred@1377750503 after retry RPC after error@1249977497) -> (continuation of Deferred@1891131966 after retry RPC after error@757296929) -> (continuation of Deferred@167181975 after retry RPC after error@1250997980)))
03:00:21.599 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.600 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.600 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.600 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.600 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.600 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.600 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.621 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.621 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.621 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.621 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.621 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.621 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.622 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.641 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.641 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.641 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.641 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.641 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.641 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.660 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.660 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.660 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.660 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.660 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.660 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.679 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.680 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.680 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.680 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.680 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.680 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.680 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.719 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.720 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.720 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.720 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.720 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.720 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.720 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.721 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.741 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.741 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.741 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.741 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.741 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.741 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.742 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.742 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@322284969(state=PENDING, result=null, callback=(continuation of Deferred@1763403028 after retry RPC after error@1337742052) -> (continuation of Deferred@1828920071 after retry RPC after error@1407560683) -> (continuation of Deferred@1633961798 after retry RPC after error@1289884412) -> (continuation of Deferred@961344324 after retry RPC after error@1191751029) -> (continuation of Deferred@68521125 after retry RPC after error@1488718633) -> (continuation of Deferred@112465321 after retry RPC after error@81097487), errback=(continuation of Deferred@1763403028 after retry RPC after error@1337742052) -> (continuation of Deferred@1828920071 after retry RPC after error@1407560683) -> (continuation of Deferred@1633961798 after retry RPC after error@1289884412) -> (continuation of Deferred@961344324 after retry RPC after error@1191751029) -> (continuation of Deferred@68521125 after retry RPC after error@1488718633) -> (continuation of Deferred@112465321 after retry RPC after error@81097487)))
03:00:21.742 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@322284969(state=PENDING, result=null, callback=(continuation of Deferred@1763403028 after retry RPC after error@1337742052) -> (continuation of Deferred@1828920071 after retry RPC after error@1407560683) -> (continuation of Deferred@1633961798 after retry RPC after error@1289884412) -> (continuation of Deferred@961344324 after retry RPC after error@1191751029) -> (continuation of Deferred@68521125 after retry RPC after error@1488718633) -> (continuation of Deferred@112465321 after retry RPC after error@81097487), errback=(continuation of Deferred@1763403028 after retry RPC after error@1337742052) -> (continuation of Deferred@1828920071 after retry RPC after error@1407560683) -> (continuation of Deferred@1633961798 after retry RPC after error@1289884412) -> (continuation of Deferred@961344324 after retry RPC after error@1191751029) -> (continuation of Deferred@68521125 after retry RPC after error@1488718633) -> (continuation of Deferred@112465321 after retry RPC after error@81097487)))
03:00:21.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.760 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.760 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.760 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.760 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.760 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.761 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.779 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.780 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.780 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.780 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.780 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.780 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.780 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.799 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.799 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.800 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.800 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.800 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.800 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.800 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.800 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.819 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.820 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.820 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.820 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.820 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.820 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.820 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.839 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.839 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.840 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.840 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.840 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.840 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.840 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.840 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.881 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.881 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.881 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.881 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.881 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.881 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.882 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.901 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.901 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.901 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.901 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.901 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.901 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.902 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:21.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:21.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:21.961 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.961 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.961 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.961 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.961 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:21.961 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:21.962 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.001 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.001 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.001 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.001 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.001 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.001 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.002 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [0ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1551705978(state=PENDING, result=null, callback=(continuation of Deferred@1209709414 after retry RPC after error@2953150) -> (continuation of Deferred@1967608353 after retry RPC after error@1085603076) -> (continuation of Deferred@1571354615 after retry RPC after error@898646787) -> (continuation of Deferred@2055854538 after retry RPC after error@489838402) -> (continuation of Deferred@129965148 after retry RPC after error@805359603) -> (continuation of Deferred@74647472 after retry RPC after error@200591419) -> (continuation of Deferred@1951820005 after retry RPC after error@1109352139) -> (continuation of Deferred@1590529983 after retry RPC after error@1943347533), errback=(continuation of Deferred@1209709414 after retry RPC after error@2953150) -> (continuation of Deferred@1967608353 after retry RPC after error@1085603076) -> (continuation of Deferred@1571354615 after retry RPC after error@898646787) -> (continuation of Deferred@2055854538 after retry RPC after error@489838402) -> (continuation of Deferred@129965148 after retry RPC after error@805359603) -> (continuation of Deferred@74647472 after retry RPC after error@200591419) -> (continuation of Deferred@1951820005 after retry RPC after error@1109352139) -> (continuation of Deferred@1590529983 after retry RPC after error@1943347533)))
03:00:22.002 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [0ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1551705978(state=PENDING, result=null, callback=(continuation of Deferred@1209709414 after retry RPC after error@2953150) -> (continuation of Deferred@1967608353 after retry RPC after error@1085603076) -> (continuation of Deferred@1571354615 after retry RPC after error@898646787) -> (continuation of Deferred@2055854538 after retry RPC after error@489838402) -> (continuation of Deferred@129965148 after retry RPC after error@805359603) -> (continuation of Deferred@74647472 after retry RPC after error@200591419) -> (continuation of Deferred@1951820005 after retry RPC after error@1109352139) -> (continuation of Deferred@1590529983 after retry RPC after error@1943347533), errback=(continuation of Deferred@1209709414 after retry RPC after error@2953150) -> (continuation of Deferred@1967608353 after retry RPC after error@1085603076) -> (continuation of Deferred@1571354615 after retry RPC after error@898646787) -> (continuation of Deferred@2055854538 after retry RPC after error@489838402) -> (continuation of Deferred@129965148 after retry RPC after error@805359603) -> (continuation of Deferred@74647472 after retry RPC after error@200591419) -> (continuation of Deferred@1951820005 after retry RPC after error@1109352139) -> (continuation of Deferred@1590529983 after retry RPC after error@1943347533)))
03:00:22.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.020 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.020 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.020 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.020 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.021 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.021 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.040 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.040 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.040 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.040 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.040 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.040 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.041 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.059 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.060 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.060 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.060 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.061 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.061 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.061 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.081 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.081 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.081 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.081 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.081 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.081 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.082 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.100 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.101 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.101 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.101 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.101 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.101 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.141 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.141 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.141 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.141 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.141 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.141 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.141 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.180 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.180 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.180 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.181 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.181 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.181 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.241 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.241 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.241 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.241 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.241 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.241 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.241 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.260 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.260 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.260 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.260 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.261 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.261 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.261 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.266 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1259197877(state=PENDING, result=null, callback=(continuation of Deferred@1408884724 after retry RPC after error@1863917615) -> (continuation of Deferred@621028039 after retry RPC after error@8518575) -> (continuation of Deferred@488276532 after retry RPC after error@1000657376) -> (continuation of Deferred@1068686921 after retry RPC after error@820053651) -> (continuation of Deferred@2040736153 after retry RPC after error@1687542940) -> (continuation of Deferred@1299973841 after retry RPC after error@1972458611) -> (continuation of Deferred@354305209 after retry RPC after error@1604656058) -> (continuation of Deferred@1925370897 after retry RPC after error@227484918), errback=(continuation of Deferred@1408884724 after retry RPC after error@1863917615) -> (continuation of Deferred@621028039 after retry RPC after error@8518575) -> (continuation of Deferred@488276532 after retry RPC after error@1000657376) -> (continuation of Deferred@1068686921 after retry RPC after error@820053651) -> (continuation of Deferred@2040736153 after retry RPC after error@1687542940) -> (continuation of Deferred@1299973841 after retry RPC after error@1972458611) -> (continuation of Deferred@354305209 after retry RPC after error@1604656058) -> (continuation of Deferred@1925370897 after retry RPC after error@227484918)))
03:00:22.266 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [240ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1259197877(state=PENDING, result=null, callback=(continuation of Deferred@1408884724 after retry RPC after error@1863917615) -> (continuation of Deferred@621028039 after retry RPC after error@8518575) -> (continuation of Deferred@488276532 after retry RPC after error@1000657376) -> (continuation of Deferred@1068686921 after retry RPC after error@820053651) -> (continuation of Deferred@2040736153 after retry RPC after error@1687542940) -> (continuation of Deferred@1299973841 after retry RPC after error@1972458611) -> (continuation of Deferred@354305209 after retry RPC after error@1604656058) -> (continuation of Deferred@1925370897 after retry RPC after error@227484918), errback=(continuation of Deferred@1408884724 after retry RPC after error@1863917615) -> (continuation of Deferred@621028039 after retry RPC after error@8518575) -> (continuation of Deferred@488276532 after retry RPC after error@1000657376) -> (continuation of Deferred@1068686921 after retry RPC after error@820053651) -> (continuation of Deferred@2040736153 after retry RPC after error@1687542940) -> (continuation of Deferred@1299973841 after retry RPC after error@1972458611) -> (continuation of Deferred@354305209 after retry RPC after error@1604656058) -> (continuation of Deferred@1925370897 after retry RPC after error@227484918)))
03:00:22.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.280 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.280 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.280 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.280 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.280 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.280 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.299 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.300 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.300 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.300 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.300 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.300 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.300 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.319 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.320 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.320 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.320 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.320 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.320 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.320 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.341 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.341 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.341 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.341 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.341 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.341 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.342 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.360 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.360 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.360 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.360 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.360 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.361 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.401 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.401 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.401 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.401 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.401 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.401 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.402 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.441 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.441 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.441 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.441 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.441 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.441 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.500 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.501 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.501 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.501 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.501 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.501 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.502 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@2075156774(state=PENDING, result=null, callback=(continuation of Deferred@133309677 after retry RPC after error@772545237) -> (continuation of Deferred@304188528 after retry RPC after error@1793095235) -> (continuation of Deferred@2091119294 after retry RPC after error@850892109) -> (continuation of Deferred@745262439 after retry RPC after error@1171322055) -> (continuation of Deferred@646305985 after retry RPC after error@680930465) -> (continuation of Deferred@1524943905 after retry RPC after error@416312694) -> (continuation of Deferred@270265725 after retry RPC after error@166803707), errback=(continuation of Deferred@133309677 after retry RPC after error@772545237) -> (continuation of Deferred@304188528 after retry RPC after error@1793095235) -> (continuation of Deferred@2091119294 after retry RPC after error@850892109) -> (continuation of Deferred@745262439 after retry RPC after error@1171322055) -> (continuation of Deferred@646305985 after retry RPC after error@680930465) -> (continuation of Deferred@1524943905 after retry RPC after error@416312694) -> (continuation of Deferred@270265725 after retry RPC after error@166803707)))
03:00:22.502 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [222ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@2075156774(state=PENDING, result=null, callback=(continuation of Deferred@133309677 after retry RPC after error@772545237) -> (continuation of Deferred@304188528 after retry RPC after error@1793095235) -> (continuation of Deferred@2091119294 after retry RPC after error@850892109) -> (continuation of Deferred@745262439 after retry RPC after error@1171322055) -> (continuation of Deferred@646305985 after retry RPC after error@680930465) -> (continuation of Deferred@1524943905 after retry RPC after error@416312694) -> (continuation of Deferred@270265725 after retry RPC after error@166803707), errback=(continuation of Deferred@133309677 after retry RPC after error@772545237) -> (continuation of Deferred@304188528 after retry RPC after error@1793095235) -> (continuation of Deferred@2091119294 after retry RPC after error@850892109) -> (continuation of Deferred@745262439 after retry RPC after error@1171322055) -> (continuation of Deferred@646305985 after retry RPC after error@680930465) -> (continuation of Deferred@1524943905 after retry RPC after error@416312694) -> (continuation of Deferred@270265725 after retry RPC after error@166803707)))
03:00:22.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.520 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.520 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.520 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.520 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.520 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.520 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.539 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.540 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.540 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.540 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.540 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.540 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.540 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.561 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.561 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.561 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.561 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.561 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.561 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.561 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.580 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.580 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.581 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.581 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.581 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.581 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.600 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.600 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.600 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.601 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.601 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.601 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:22.611745 21405 raft_consensus.cc:534] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 56099a6022da440a81f20e6fe8903fb0)
03:00:22.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:22.611964 21405 raft_consensus.cc:556] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
03:00:22.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:22.612876 21405 leader_election.cc:290] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
03:00:22.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 03:00:22.613992 20381 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111) [suppressed 3 similar messages]
03:00:22.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 03:00:22.615649 20378 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111): Network error: Client connection negotiation failed: client connection to 127.19.203.254:46111: connect: Connection refused (error 111)
03:00:22.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 03:00:22.615837 20381 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
03:00:22.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:22.615996 20381 leader_election.cc:304] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c; no voters: 56099a6022da440a81f20e6fe8903fb0, efb4c69b20bd4f9692d496c33ddbb24e
03:00:22.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:22.616277 21405 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
03:00:22.619 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.620 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.620 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.620 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.620 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.620 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.620 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.660 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.660 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.660 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.660 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.660 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.660 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.720 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.720 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.720 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.720 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.720 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.721 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.721 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.722 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@400465874(state=PENDING, result=null, callback=(continuation of Deferred@962159811 after retry RPC after error@588312639) -> (continuation of Deferred@256428601 after retry RPC after error@1821111309) -> (continuation of Deferred@512162339 after retry RPC after error@984834244) -> (continuation of Deferred@1935895721 after retry RPC after error@1081857364) -> (continuation of Deferred@1761962138 after retry RPC after error@1217499198) -> (continuation of Deferred@553630625 after retry RPC after error@1051728799) -> (continuation of Deferred@2092168555 after retry RPC after error@946313891), errback=(continuation of Deferred@962159811 after retry RPC after error@588312639) -> (continuation of Deferred@256428601 after retry RPC after error@1821111309) -> (continuation of Deferred@512162339 after retry RPC after error@984834244) -> (continuation of Deferred@1935895721 after retry RPC after error@1081857364) -> (continuation of Deferred@1761962138 after retry RPC after error@1217499198) -> (continuation of Deferred@553630625 after retry RPC after error@1051728799) -> (continuation of Deferred@2092168555 after retry RPC after error@946313891)))
03:00:22.722 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@400465874(state=PENDING, result=null, callback=(continuation of Deferred@962159811 after retry RPC after error@588312639) -> (continuation of Deferred@256428601 after retry RPC after error@1821111309) -> (continuation of Deferred@512162339 after retry RPC after error@984834244) -> (continuation of Deferred@1935895721 after retry RPC after error@1081857364) -> (continuation of Deferred@1761962138 after retry RPC after error@1217499198) -> (continuation of Deferred@553630625 after retry RPC after error@1051728799) -> (continuation of Deferred@2092168555 after retry RPC after error@946313891), errback=(continuation of Deferred@962159811 after retry RPC after error@588312639) -> (continuation of Deferred@256428601 after retry RPC after error@1821111309) -> (continuation of Deferred@512162339 after retry RPC after error@984834244) -> (continuation of Deferred@1935895721 after retry RPC after error@1081857364) -> (continuation of Deferred@1761962138 after retry RPC after error@1217499198) -> (continuation of Deferred@553630625 after retry RPC after error@1051728799) -> (continuation of Deferred@2092168555 after retry RPC after error@946313891)))
03:00:22.739 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.740 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.740 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.740 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.740 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.740 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.740 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.759 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.759 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.760 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.760 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.760 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.760 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.760 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.760 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.781 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.781 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.781 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.781 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.781 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.781 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.782 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.800 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.800 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.800 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.800 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.800 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.800 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.819 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.820 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.820 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.820 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.820 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.820 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.820 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.841 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.841 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.841 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.841 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.841 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.841 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.860 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.860 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.861 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.861 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.861 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.861 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.862 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.921 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.921 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.921 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.921 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.921 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.921 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.922 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@375469413(state=PENDING, result=null, callback=(continuation of Deferred@661606717 after retry RPC after error@593175532) -> (continuation of Deferred@1109666212 after retry RPC after error@1576134296) -> (continuation of Deferred@4891298 after retry RPC after error@2054468868) -> (continuation of Deferred@436128487 after retry RPC after error@1265183334) -> (continuation of Deferred@518245915 after retry RPC after error@1096848470) -> (continuation of Deferred@774297455 after retry RPC after error@740225751) -> (continuation of Deferred@739078745 after retry RPC after error@1762296234), errback=(continuation of Deferred@661606717 after retry RPC after error@593175532) -> (continuation of Deferred@1109666212 after retry RPC after error@1576134296) -> (continuation of Deferred@4891298 after retry RPC after error@2054468868) -> (continuation of Deferred@436128487 after retry RPC after error@1265183334) -> (continuation of Deferred@518245915 after retry RPC after error@1096848470) -> (continuation of Deferred@774297455 after retry RPC after error@740225751) -> (continuation of Deferred@739078745 after retry RPC after error@1762296234)))
03:00:22.922 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@375469413(state=PENDING, result=null, callback=(continuation of Deferred@661606717 after retry RPC after error@593175532) -> (continuation of Deferred@1109666212 after retry RPC after error@1576134296) -> (continuation of Deferred@4891298 after retry RPC after error@2054468868) -> (continuation of Deferred@436128487 after retry RPC after error@1265183334) -> (continuation of Deferred@518245915 after retry RPC after error@1096848470) -> (continuation of Deferred@774297455 after retry RPC after error@740225751) -> (continuation of Deferred@739078745 after retry RPC after error@1762296234), errback=(continuation of Deferred@661606717 after retry RPC after error@593175532) -> (continuation of Deferred@1109666212 after retry RPC after error@1576134296) -> (continuation of Deferred@4891298 after retry RPC after error@2054468868) -> (continuation of Deferred@436128487 after retry RPC after error@1265183334) -> (continuation of Deferred@518245915 after retry RPC after error@1096848470) -> (continuation of Deferred@774297455 after retry RPC after error@740225751) -> (continuation of Deferred@739078745 after retry RPC after error@1762296234)))
03:00:22.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.940 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.940 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.940 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.940 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.940 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.940 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.959 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.960 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.960 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.960 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.960 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.960 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.960 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:22.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:22.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:22.981 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.981 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.981 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:22.981 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.981 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.981 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:22.982 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.000 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.000 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.000 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.001 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.001 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.001 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.020 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.020 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.020 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.020 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.020 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.020 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.039 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.039 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.040 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.040 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.040 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.040 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.040 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.040 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.041 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.099 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.099 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.100 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.100 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.100 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.100 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.100 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.100 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.101 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1557071918(state=PENDING, result=null, callback=(continuation of Deferred@365756954 after retry RPC after error@1511271494) -> (continuation of Deferred@2126671017 after retry RPC after error@1516799271) -> (continuation of Deferred@667455906 after retry RPC after error@1548087041) -> (continuation of Deferred@823092996 after retry RPC after error@802196927) -> (continuation of Deferred@1282821222 after retry RPC after error@1060671246) -> (continuation of Deferred@1975904426 after retry RPC after error@1590898392), errback=(continuation of Deferred@365756954 after retry RPC after error@1511271494) -> (continuation of Deferred@2126671017 after retry RPC after error@1516799271) -> (continuation of Deferred@667455906 after retry RPC after error@1548087041) -> (continuation of Deferred@823092996 after retry RPC after error@802196927) -> (continuation of Deferred@1282821222 after retry RPC after error@1060671246) -> (continuation of Deferred@1975904426 after retry RPC after error@1590898392)))
03:00:23.101 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1557071918(state=PENDING, result=null, callback=(continuation of Deferred@365756954 after retry RPC after error@1511271494) -> (continuation of Deferred@2126671017 after retry RPC after error@1516799271) -> (continuation of Deferred@667455906 after retry RPC after error@1548087041) -> (continuation of Deferred@823092996 after retry RPC after error@802196927) -> (continuation of Deferred@1282821222 after retry RPC after error@1060671246) -> (continuation of Deferred@1975904426 after retry RPC after error@1590898392), errback=(continuation of Deferred@365756954 after retry RPC after error@1511271494) -> (continuation of Deferred@2126671017 after retry RPC after error@1516799271) -> (continuation of Deferred@667455906 after retry RPC after error@1548087041) -> (continuation of Deferred@823092996 after retry RPC after error@802196927) -> (continuation of Deferred@1282821222 after retry RPC after error@1060671246) -> (continuation of Deferred@1975904426 after retry RPC after error@1590898392)))
03:00:23.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.121 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.121 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.121 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.121 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.121 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.121 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.122 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.140 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.140 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.140 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.140 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.140 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.140 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.141 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.159 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.160 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.160 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.160 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.160 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.160 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.160 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.160 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.181 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.181 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.181 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.181 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.182 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.182 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.182 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.182 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.200 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.200 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.200 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.201 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.201 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.201 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.220 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.220 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.220 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.220 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.220 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.220 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.260 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.260 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.260 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.261 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.261 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.261 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.261 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.361 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.361 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.361 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.361 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.362 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.362 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.362 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.362 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.362 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@943577938(state=PENDING, result=null, callback=(continuation of Deferred@572401133 after retry RPC after error@2021337982) -> (continuation of Deferred@1464743439 after retry RPC after error@25133311) -> (continuation of Deferred@1377696880 after retry RPC after error@1931218326) -> (continuation of Deferred@1430565812 after retry RPC after error@683780507) -> (continuation of Deferred@360853135 after retry RPC after error@107082004) -> (continuation of Deferred@1984445311 after retry RPC after error@1407244636) -> (continuation of Deferred@1469211641 after retry RPC after error@1584582990), errback=(continuation of Deferred@572401133 after retry RPC after error@2021337982) -> (continuation of Deferred@1464743439 after retry RPC after error@25133311) -> (continuation of Deferred@1377696880 after retry RPC after error@1931218326) -> (continuation of Deferred@1430565812 after retry RPC after error@683780507) -> (continuation of Deferred@360853135 after retry RPC after error@107082004) -> (continuation of Deferred@1984445311 after retry RPC after error@1407244636) -> (continuation of Deferred@1469211641 after retry RPC after error@1584582990)))
03:00:23.362 [DEBUG - kudu-nio-2] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@943577938(state=PENDING, result=null, callback=(continuation of Deferred@572401133 after retry RPC after error@2021337982) -> (continuation of Deferred@1464743439 after retry RPC after error@25133311) -> (continuation of Deferred@1377696880 after retry RPC after error@1931218326) -> (continuation of Deferred@1430565812 after retry RPC after error@683780507) -> (continuation of Deferred@360853135 after retry RPC after error@107082004) -> (continuation of Deferred@1984445311 after retry RPC after error@1407244636) -> (continuation of Deferred@1469211641 after retry RPC after error@1584582990), errback=(continuation of Deferred@572401133 after retry RPC after error@2021337982) -> (continuation of Deferred@1464743439 after retry RPC after error@25133311) -> (continuation of Deferred@1377696880 after retry RPC after error@1931218326) -> (continuation of Deferred@1430565812 after retry RPC after error@683780507) -> (continuation of Deferred@360853135 after retry RPC after error@107082004) -> (continuation of Deferred@1984445311 after retry RPC after error@1407244636) -> (continuation of Deferred@1469211641 after retry RPC after error@1584582990)))
03:00:23.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.380 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.380 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.381 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.381 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.381 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.381 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.400 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.400 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.400 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.400 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.400 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.400 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.419 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.420 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.420 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.420 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.420 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.420 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.420 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.421 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.441 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.441 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.451 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.451 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.451 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.451 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.451 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.481 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.481 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.481 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.481 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.481 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.481 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.481 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.500 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.500 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.500 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.500 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.500 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.501 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.519 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.520 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.520 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.520 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.520 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.520 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.520 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.640 [DEBUG - pool-40-thread-1] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [71ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [71ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [71ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@708332481(state=PENDING, result=null, callback=(continuation of Deferred@917381401 after retry RPC after error@2006438816) -> (continuation of Deferred@1105406798 after retry RPC after error@703399664) -> (continuation of Deferred@1854688317 after retry RPC after error@659523743) -> (continuation of Deferred@2142137955 after retry RPC after error@1043676640) -> (continuation of Deferred@844395455 after retry RPC after error@1761877530) -> (continuation of Deferred@1708501039 after retry RPC after error@1922585416) -> (continuation of Deferred@1665146671 after retry RPC after error@712845715), errback=(continuation of Deferred@917381401 after retry RPC after error@2006438816) -> (continuation of Deferred@1105406798 after retry RPC after error@703399664) -> (continuation of Deferred@1854688317 after retry RPC after error@659523743) -> (continuation of Deferred@2142137955 after retry RPC after error@1043676640) -> (continuation of Deferred@844395455 after retry RPC after error@1761877530) -> (continuation of Deferred@1708501039 after retry RPC after error@1922585416) -> (continuation of Deferred@1665146671 after retry RPC after error@712845715)))
03:00:23.640 [WARN - pool-40-thread-1] (HashedWheelTimer.java:729) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
03:00:23.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.661 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.661 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.661 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.661 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.661 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.661 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.662 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.680 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.680 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.680 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.681 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.681 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.681 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.699 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.700 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.700 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.700 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.700 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.700 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.700 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.719 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.719 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.720 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.720 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.720 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.720 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.720 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.720 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.721 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.740 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.740 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.740 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.741 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.741 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.741 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.760 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.760 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.760 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.760 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.760 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.760 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.819 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.820 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.820 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.820 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.820 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.820 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.820 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.820 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.820 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:23.861459 21412 raft_consensus.cc:534] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 56099a6022da440a81f20e6fe8903fb0)
03:00:23.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:23.861668 21412 raft_consensus.cc:556] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56099a6022da440a81f20e6fe8903fb0" member_type: VOTER last_known_addr { host: "127.19.203.254" port: 46111 } } peers { permanent_uuid: "1e3c2fda688743cfb0f78de4c81a953c" member_type: VOTER last_known_addr { host: "127.19.203.253" port: 34409 } } peers { permanent_uuid: "efb4c69b20bd4f9692d496c33ddbb24e" member_type: VOTER last_known_addr { host: "127.19.203.252" port: 40563 } }
03:00:23.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:23.862325 21412 leader_election.cc:290] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111), efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563)
03:00:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 03:00:23.864621 20378 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 56099a6022da440a81f20e6fe8903fb0 (127.19.203.254:46111): Network error: Client connection negotiation failed: client connection to 127.19.203.254:46111: connect: Connection refused (error 111)
03:00:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261121 03:00:23.864830 20381 leader_election.cc:336] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer efb4c69b20bd4f9692d496c33ddbb24e (127.19.203.252:40563): Network error: Client connection negotiation failed: client connection to 127.19.203.252:40563: connect: Connection refused (error 111)
03:00:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:23.865017 20381 leader_election.cc:304] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1e3c2fda688743cfb0f78de4c81a953c; no voters: 56099a6022da440a81f20e6fe8903fb0, efb4c69b20bd4f9692d496c33ddbb24e
03:00:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:23.865320 21412 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 1e3c2fda688743cfb0f78de4c81a953c [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
03:00:23.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.900 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.900 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.901 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.901 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.901 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.901 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.901 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1655007187(state=PENDING, result=null, callback=(continuation of Deferred@1391216489 after retry RPC after error@449709673) -> (continuation of Deferred@1215238899 after retry RPC after error@29192146) -> (continuation of Deferred@836413315 after retry RPC after error@1540166303) -> (continuation of Deferred@1803910047 after retry RPC after error@19679938) -> (continuation of Deferred@1697518555 after retry RPC after error@1455733771) -> (continuation of Deferred@383610063 after retry RPC after error@1975799436) -> (continuation of Deferred@1625373476 after retry RPC after error@330754007), errback=(continuation of Deferred@1391216489 after retry RPC after error@449709673) -> (continuation of Deferred@1215238899 after retry RPC after error@29192146) -> (continuation of Deferred@836413315 after retry RPC after error@1540166303) -> (continuation of Deferred@1803910047 after retry RPC after error@19679938) -> (continuation of Deferred@1697518555 after retry RPC after error@1455733771) -> (continuation of Deferred@383610063 after retry RPC after error@1975799436) -> (continuation of Deferred@1625373476 after retry RPC after error@330754007)))
03:00:23.902 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1655007187(state=PENDING, result=null, callback=(continuation of Deferred@1391216489 after retry RPC after error@449709673) -> (continuation of Deferred@1215238899 after retry RPC after error@29192146) -> (continuation of Deferred@836413315 after retry RPC after error@1540166303) -> (continuation of Deferred@1803910047 after retry RPC after error@19679938) -> (continuation of Deferred@1697518555 after retry RPC after error@1455733771) -> (continuation of Deferred@383610063 after retry RPC after error@1975799436) -> (continuation of Deferred@1625373476 after retry RPC after error@330754007), errback=(continuation of Deferred@1391216489 after retry RPC after error@449709673) -> (continuation of Deferred@1215238899 after retry RPC after error@29192146) -> (continuation of Deferred@836413315 after retry RPC after error@1540166303) -> (continuation of Deferred@1803910047 after retry RPC after error@19679938) -> (continuation of Deferred@1697518555 after retry RPC after error@1455733771) -> (continuation of Deferred@383610063 after retry RPC after error@1975799436) -> (continuation of Deferred@1625373476 after retry RPC after error@330754007)))
03:00:23.919 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.920 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.920 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.920 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.920 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.920 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.920 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.939 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.939 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.940 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.940 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.940 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.940 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.940 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.940 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.961 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.961 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.961 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.961 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.961 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.961 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.961 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.979 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:23.980 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.980 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.980 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.980 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.980 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:23.980 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:23.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:23.999 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:23.999 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.000 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.000 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.000 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.000 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.000 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.000 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.021 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.021 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.021 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.021 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.021 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.021 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.060 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.060 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.061 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.061 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.061 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.061 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.159 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.159 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.160 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.160 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.160 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.160 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.160 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.160 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.161 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.161 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@413505526(state=PENDING, result=null, callback=(continuation of Deferred@528298525 after retry RPC after error@2016721628) -> (continuation of Deferred@1577034537 after retry RPC after error@1775338823) -> (continuation of Deferred@524988346 after retry RPC after error@1114965205) -> (continuation of Deferred@1655898457 after retry RPC after error@120409077) -> (continuation of Deferred@1305478101 after retry RPC after error@661251350) -> (continuation of Deferred@1563070827 after retry RPC after error@808786904) -> (continuation of Deferred@2109106336 after retry RPC after error@362767189), errback=(continuation of Deferred@528298525 after retry RPC after error@2016721628) -> (continuation of Deferred@1577034537 after retry RPC after error@1775338823) -> (continuation of Deferred@524988346 after retry RPC after error@1114965205) -> (continuation of Deferred@1655898457 after retry RPC after error@120409077) -> (continuation of Deferred@1305478101 after retry RPC after error@661251350) -> (continuation of Deferred@1563070827 after retry RPC after error@808786904) -> (continuation of Deferred@2109106336 after retry RPC after error@362767189)))
03:00:24.161 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@413505526(state=PENDING, result=null, callback=(continuation of Deferred@528298525 after retry RPC after error@2016721628) -> (continuation of Deferred@1577034537 after retry RPC after error@1775338823) -> (continuation of Deferred@524988346 after retry RPC after error@1114965205) -> (continuation of Deferred@1655898457 after retry RPC after error@120409077) -> (continuation of Deferred@1305478101 after retry RPC after error@661251350) -> (continuation of Deferred@1563070827 after retry RPC after error@808786904) -> (continuation of Deferred@2109106336 after retry RPC after error@362767189), errback=(continuation of Deferred@528298525 after retry RPC after error@2016721628) -> (continuation of Deferred@1577034537 after retry RPC after error@1775338823) -> (continuation of Deferred@524988346 after retry RPC after error@1114965205) -> (continuation of Deferred@1655898457 after retry RPC after error@120409077) -> (continuation of Deferred@1305478101 after retry RPC after error@661251350) -> (continuation of Deferred@1563070827 after retry RPC after error@808786904) -> (continuation of Deferred@2109106336 after retry RPC after error@362767189)))
03:00:24.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.181 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.181 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.181 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.181 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.181 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.181 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.200 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.200 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.200 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.200 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.200 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.200 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.219 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.220 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.220 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.220 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.220 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.220 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.220 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.239 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.240 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.240 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.240 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.240 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.240 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.240 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.241 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.261 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.261 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.261 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.261 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.261 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.261 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.262 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.301 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.301 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.302 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.302 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.302 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.302 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.302 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.341 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.341 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.341 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.341 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.341 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.341 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.342 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1257877905(state=PENDING, result=null, callback=(continuation of Deferred@605222480 after retry RPC after error@13350575) -> (continuation of Deferred@637277893 after retry RPC after error@571363026) -> (continuation of Deferred@287153333 after retry RPC after error@1287536375) -> (continuation of Deferred@162612123 after retry RPC after error@1503257761) -> (continuation of Deferred@605823763 after retry RPC after error@434694369) -> (continuation of Deferred@449236444 after retry RPC after error@216833737), errback=(continuation of Deferred@605222480 after retry RPC after error@13350575) -> (continuation of Deferred@637277893 after retry RPC after error@571363026) -> (continuation of Deferred@287153333 after retry RPC after error@1287536375) -> (continuation of Deferred@162612123 after retry RPC after error@1503257761) -> (continuation of Deferred@605823763 after retry RPC after error@434694369) -> (continuation of Deferred@449236444 after retry RPC after error@216833737)))
03:00:24.342 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1257877905(state=PENDING, result=null, callback=(continuation of Deferred@605222480 after retry RPC after error@13350575) -> (continuation of Deferred@637277893 after retry RPC after error@571363026) -> (continuation of Deferred@287153333 after retry RPC after error@1287536375) -> (continuation of Deferred@162612123 after retry RPC after error@1503257761) -> (continuation of Deferred@605823763 after retry RPC after error@434694369) -> (continuation of Deferred@449236444 after retry RPC after error@216833737), errback=(continuation of Deferred@605222480 after retry RPC after error@13350575) -> (continuation of Deferred@637277893 after retry RPC after error@571363026) -> (continuation of Deferred@287153333 after retry RPC after error@1287536375) -> (continuation of Deferred@162612123 after retry RPC after error@1503257761) -> (continuation of Deferred@605823763 after retry RPC after error@434694369) -> (continuation of Deferred@449236444 after retry RPC after error@216833737)))
03:00:24.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.360 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.360 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.361 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.361 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.361 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.361 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.380 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.380 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.380 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.380 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.380 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.380 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.399 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.400 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.400 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.400 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.400 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.400 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.400 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.420 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.421 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.421 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.421 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.421 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.421 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.421 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.421 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.422 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.439 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.440 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.440 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.440 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.440 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.440 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.440 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.479 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.480 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.480 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.480 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.480 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.480 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.480 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.481 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.539 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.540 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.540 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.540 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.540 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.540 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.540 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.559 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.559 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.560 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.560 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.560 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.560 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.560 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.560 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.561 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.561 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1939884783(state=PENDING, result=null, callback=(continuation of Deferred@121510133 after retry RPC after error@2073313057) -> (continuation of Deferred@2036975103 after retry RPC after error@149884687) -> (continuation of Deferred@2145470295 after retry RPC after error@626393789) -> (continuation of Deferred@304578005 after retry RPC after error@1656791017) -> (continuation of Deferred@401520409 after retry RPC after error@37227699) -> (continuation of Deferred@735580404 after retry RPC after error@1300364721) -> (continuation of Deferred@792059239 after retry RPC after error@1839858086), errback=(continuation of Deferred@121510133 after retry RPC after error@2073313057) -> (continuation of Deferred@2036975103 after retry RPC after error@149884687) -> (continuation of Deferred@2145470295 after retry RPC after error@626393789) -> (continuation of Deferred@304578005 after retry RPC after error@1656791017) -> (continuation of Deferred@401520409 after retry RPC after error@37227699) -> (continuation of Deferred@735580404 after retry RPC after error@1300364721) -> (continuation of Deferred@792059239 after retry RPC after error@1839858086)))
03:00:24.561 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1939884783(state=PENDING, result=null, callback=(continuation of Deferred@121510133 after retry RPC after error@2073313057) -> (continuation of Deferred@2036975103 after retry RPC after error@149884687) -> (continuation of Deferred@2145470295 after retry RPC after error@626393789) -> (continuation of Deferred@304578005 after retry RPC after error@1656791017) -> (continuation of Deferred@401520409 after retry RPC after error@37227699) -> (continuation of Deferred@735580404 after retry RPC after error@1300364721) -> (continuation of Deferred@792059239 after retry RPC after error@1839858086), errback=(continuation of Deferred@121510133 after retry RPC after error@2073313057) -> (continuation of Deferred@2036975103 after retry RPC after error@149884687) -> (continuation of Deferred@2145470295 after retry RPC after error@626393789) -> (continuation of Deferred@304578005 after retry RPC after error@1656791017) -> (continuation of Deferred@401520409 after retry RPC after error@37227699) -> (continuation of Deferred@735580404 after retry RPC after error@1300364721) -> (continuation of Deferred@792059239 after retry RPC after error@1839858086)))
03:00:24.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.581 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.581 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.581 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.581 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.581 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.581 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.600 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.600 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.600 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.600 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.600 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.600 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.619 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.620 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.620 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.620 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.620 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.620 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.620 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.640 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.641 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.641 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.641 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.641 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.641 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.641 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.642 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.660 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.660 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.660 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.660 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.661 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.661 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.661 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.680 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.680 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.680 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.680 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.680 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.680 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.740 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.740 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.740 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.740 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.740 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.740 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.779 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.780 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.780 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.780 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.780 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.780 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.780 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.781 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@150996860(state=PENDING, result=null, callback=(continuation of Deferred@1243684454 after retry RPC after error@1900006175) -> (continuation of Deferred@589511921 after retry RPC after error@714204174) -> (continuation of Deferred@1753287956 after retry RPC after error@1734956072) -> (continuation of Deferred@1570191488 after retry RPC after error@2013464232) -> (continuation of Deferred@411720977 after retry RPC after error@904892988) -> (continuation of Deferred@1373175609 after retry RPC after error@823035372) -> (continuation of Deferred@2074485769 after retry RPC after error@848373555), errback=(continuation of Deferred@1243684454 after retry RPC after error@1900006175) -> (continuation of Deferred@589511921 after retry RPC after error@714204174) -> (continuation of Deferred@1753287956 after retry RPC after error@1734956072) -> (continuation of Deferred@1570191488 after retry RPC after error@2013464232) -> (continuation of Deferred@411720977 after retry RPC after error@904892988) -> (continuation of Deferred@1373175609 after retry RPC after error@823035372) -> (continuation of Deferred@2074485769 after retry RPC after error@848373555)))
03:00:24.782 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [200ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@150996860(state=PENDING, result=null, callback=(continuation of Deferred@1243684454 after retry RPC after error@1900006175) -> (continuation of Deferred@589511921 after retry RPC after error@714204174) -> (continuation of Deferred@1753287956 after retry RPC after error@1734956072) -> (continuation of Deferred@1570191488 after retry RPC after error@2013464232) -> (continuation of Deferred@411720977 after retry RPC after error@904892988) -> (continuation of Deferred@1373175609 after retry RPC after error@823035372) -> (continuation of Deferred@2074485769 after retry RPC after error@848373555), errback=(continuation of Deferred@1243684454 after retry RPC after error@1900006175) -> (continuation of Deferred@589511921 after retry RPC after error@714204174) -> (continuation of Deferred@1753287956 after retry RPC after error@1734956072) -> (continuation of Deferred@1570191488 after retry RPC after error@2013464232) -> (continuation of Deferred@411720977 after retry RPC after error@904892988) -> (continuation of Deferred@1373175609 after retry RPC after error@823035372) -> (continuation of Deferred@2074485769 after retry RPC after error@848373555)))
03:00:24.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.801 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.801 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.801 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.801 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.801 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.801 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.801 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.802 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.821 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.821 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.821 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.821 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.821 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.821 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.840 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.840 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.840 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.840 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.841 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.841 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.859 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.860 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.860 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.860 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.860 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.860 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.860 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.879 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.879 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.880 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.880 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.880 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.880 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.880 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.880 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.900 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.900 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.901 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.901 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.901 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.901 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.901 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.940 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.941 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.941 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.941 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.941 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.941 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.960 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.960 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.960 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.960 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.960 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.960 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.979 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:24.979 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:24.980 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.980 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.980 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.980 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.980 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:24.980 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:24.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:24.981 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=180), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@684076531(state=PENDING, result=null, callback=(continuation of Deferred@1345924154 after retry RPC after error@2065494807) -> (continuation of Deferred@2127123803 after retry RPC after error@1736001220) -> (continuation of Deferred@2126049005 after retry RPC after error@1646280294) -> (continuation of Deferred@1450691828 after retry RPC after error@2073427864) -> (continuation of Deferred@1113831809 after retry RPC after error@559115429) -> (continuation of Deferred@1227555863 after retry RPC after error@496389013) -> (continuation of Deferred@1114002967 after retry RPC after error@2020224186) -> (continuation of Deferred@1777374236 after retry RPC after error@1184211028), errback=(continuation of Deferred@1345924154 after retry RPC after error@2065494807) -> (continuation of Deferred@2127123803 after retry RPC after error@1736001220) -> (continuation of Deferred@2126049005 after retry RPC after error@1646280294) -> (continuation of Deferred@1450691828 after retry RPC after error@2073427864) -> (continuation of Deferred@1113831809 after retry RPC after error@559115429) -> (continuation of Deferred@1227555863 after retry RPC after error@496389013) -> (continuation of Deferred@1114002967 after retry RPC after error@2020224186) -> (continuation of Deferred@1777374236 after retry RPC after error@1184211028)))
03:00:24.981 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=180), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@684076531(state=PENDING, result=null, callback=(continuation of Deferred@1345924154 after retry RPC after error@2065494807) -> (continuation of Deferred@2127123803 after retry RPC after error@1736001220) -> (continuation of Deferred@2126049005 after retry RPC after error@1646280294) -> (continuation of Deferred@1450691828 after retry RPC after error@2073427864) -> (continuation of Deferred@1113831809 after retry RPC after error@559115429) -> (continuation of Deferred@1227555863 after retry RPC after error@496389013) -> (continuation of Deferred@1114002967 after retry RPC after error@2020224186) -> (continuation of Deferred@1777374236 after retry RPC after error@1184211028), errback=(continuation of Deferred@1345924154 after retry RPC after error@2065494807) -> (continuation of Deferred@2127123803 after retry RPC after error@1736001220) -> (continuation of Deferred@2126049005 after retry RPC after error@1646280294) -> (continuation of Deferred@1450691828 after retry RPC after error@2073427864) -> (continuation of Deferred@1113831809 after retry RPC after error@559115429) -> (continuation of Deferred@1227555863 after retry RPC after error@496389013) -> (continuation of Deferred@1114002967 after retry RPC after error@2020224186) -> (continuation of Deferred@1777374236 after retry RPC after error@1184211028)))
03:00:25.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.001 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.001 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.001 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.001 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.001 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.001 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.021 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.021 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.021 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.021 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.021 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.021 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.040 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.040 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.040 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.040 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.040 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.040 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.042 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.059 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.060 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.060 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.060 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.060 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.060 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.060 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.081 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.081 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.081 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.081 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.081 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.081 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.082 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.120 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.121 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.121 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.121 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.121 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.121 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.121 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.121 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.122 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.179 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.180 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.180 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.180 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.180 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.180 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.180 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.180 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.219 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.220 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.220 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.220 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.220 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.220 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.220 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.220 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.260 [DEBUG - pool-40-thread-1] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1708233754(state=PENDING, result=null, callback=(continuation of Deferred@1561541591 after retry RPC after error@1452540787) -> (continuation of Deferred@1925941796 after retry RPC after error@466854179) -> (continuation of Deferred@1608186893 after retry RPC after error@690217758) -> (continuation of Deferred@119002804 after retry RPC after error@30123468) -> (continuation of Deferred@158931820 after retry RPC after error@1064579662) -> (continuation of Deferred@116384098 after retry RPC after error@1524023734) -> (continuation of Deferred@464847867 after retry RPC after error@97363073) -> (continuation of Deferred@2085659051 after retry RPC after error@283579622), errback=(continuation of Deferred@1561541591 after retry RPC after error@1452540787) -> (continuation of Deferred@1925941796 after retry RPC after error@466854179) -> (continuation of Deferred@1608186893 after retry RPC after error@690217758) -> (continuation of Deferred@119002804 after retry RPC after error@30123468) -> (continuation of Deferred@158931820 after retry RPC after error@1064579662) -> (continuation of Deferred@116384098 after retry RPC after error@1524023734) -> (continuation of Deferred@464847867 after retry RPC after error@97363073) -> (continuation of Deferred@2085659051 after retry RPC after error@283579622)))
03:00:25.261 [WARN - pool-40-thread-1] (HashedWheelTimer.java:729) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
03:00:25.279 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.280 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.280 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.280 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.280 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.280 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.280 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.280 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.299 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.299 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.300 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.300 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.300 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.300 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.300 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.300 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.320 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.320 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.320 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.321 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.321 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.321 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.321 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.340 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.340 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.340 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.340 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.340 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.340 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.359 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.360 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.360 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.360 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.360 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.360 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.360 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.360 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.399 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.400 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.400 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.400 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.400 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.400 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.400 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.400 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.459 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.460 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.460 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.460 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.460 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.460 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.460 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.461 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.481 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.481 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.481 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.481 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.481 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.481 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.482 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.482 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [203ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1274288093(state=PENDING, result=null, callback=(continuation of Deferred@1228694992 after retry RPC after error@630914586) -> (continuation of Deferred@1965263216 after retry RPC after error@503456502) -> (continuation of Deferred@884655619 after retry RPC after error@1185055904) -> (continuation of Deferred@790883597 after retry RPC after error@549027725) -> (continuation of Deferred@1121005383 after retry RPC after error@268710969) -> (continuation of Deferred@553642161 after retry RPC after error@1263648308) -> (continuation of Deferred@11606798 after retry RPC after error@832918237), errback=(continuation of Deferred@1228694992 after retry RPC after error@630914586) -> (continuation of Deferred@1965263216 after retry RPC after error@503456502) -> (continuation of Deferred@884655619 after retry RPC after error@1185055904) -> (continuation of Deferred@790883597 after retry RPC after error@549027725) -> (continuation of Deferred@1121005383 after retry RPC after error@268710969) -> (continuation of Deferred@553642161 after retry RPC after error@1263648308) -> (continuation of Deferred@11606798 after retry RPC after error@832918237)))
03:00:25.482 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [122ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [203ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1274288093(state=PENDING, result=null, callback=(continuation of Deferred@1228694992 after retry RPC after error@630914586) -> (continuation of Deferred@1965263216 after retry RPC after error@503456502) -> (continuation of Deferred@884655619 after retry RPC after error@1185055904) -> (continuation of Deferred@790883597 after retry RPC after error@549027725) -> (continuation of Deferred@1121005383 after retry RPC after error@268710969) -> (continuation of Deferred@553642161 after retry RPC after error@1263648308) -> (continuation of Deferred@11606798 after retry RPC after error@832918237), errback=(continuation of Deferred@1228694992 after retry RPC after error@630914586) -> (continuation of Deferred@1965263216 after retry RPC after error@503456502) -> (continuation of Deferred@884655619 after retry RPC after error@1185055904) -> (continuation of Deferred@790883597 after retry RPC after error@549027725) -> (continuation of Deferred@1121005383 after retry RPC after error@268710969) -> (continuation of Deferred@553642161 after retry RPC after error@1263648308) -> (continuation of Deferred@11606798 after retry RPC after error@832918237)))
03:00:25.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.500 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.501 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.501 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.501 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.501 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.501 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.520 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.520 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.520 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.520 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.520 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.520 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.539 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.540 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.540 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.540 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.540 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.540 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.540 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.561 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.561 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.561 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.561 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.561 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.561 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.562 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.580 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.580 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.580 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.581 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.581 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.581 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.581 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.620 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.620 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.620 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.620 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.620 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.620 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.621 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.680 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.680 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.680 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.680 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.680 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.680 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.760 [DEBUG - pool-40-thread-1] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [120ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [121ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [180ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@110266093(state=PENDING, result=null, callback=(continuation of Deferred@49947293 after retry RPC after error@2008069249) -> (continuation of Deferred@1700068156 after retry RPC after error@2003025045) -> (continuation of Deferred@610174510 after retry RPC after error@1446521496) -> (continuation of Deferred@343389816 after retry RPC after error@2115647351) -> (continuation of Deferred@782606700 after retry RPC after error@962474698) -> (continuation of Deferred@280201199 after retry RPC after error@1069394689) -> (continuation of Deferred@1016124251 after retry RPC after error@1955626593), errback=(continuation of Deferred@49947293 after retry RPC after error@2008069249) -> (continuation of Deferred@1700068156 after retry RPC after error@2003025045) -> (continuation of Deferred@610174510 after retry RPC after error@1446521496) -> (continuation of Deferred@343389816 after retry RPC after error@2115647351) -> (continuation of Deferred@782606700 after retry RPC after error@962474698) -> (continuation of Deferred@280201199 after retry RPC after error@1069394689) -> (continuation of Deferred@1016124251 after retry RPC after error@1955626593)))
03:00:25.760 [WARN - pool-40-thread-1] (HashedWheelTimer.java:729) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:726)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:714)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:800)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:514)
at java.base/java.lang.Thread.run(Thread.java:840)
03:00:25.779 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.779 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.780 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.780 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.780 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.780 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.780 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.780 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.800 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.800 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.801 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.801 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.801 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.801 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.820 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.820 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.820 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.820 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.820 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.820 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.839 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.839 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.840 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.840 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.840 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.840 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.840 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.840 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.861 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.861 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.861 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.861 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.861 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.861 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.880 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.880 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.880 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.880 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.881 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.881 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.920 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.920 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.920 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.921 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.921 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.921 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.939 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.940 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.940 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.940 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.941 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.941 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.941 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.941 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.960 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.961 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.961 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.961 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.961 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.961 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:25.980 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:25.980 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.980 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.980 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:25.980 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.980 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.981 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:25.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:25.981 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=10, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1690348130(state=PENDING, result=null, callback=(continuation of Deferred@121946548 after retry RPC after error@574861958) -> (continuation of Deferred@2135465286 after retry RPC after error@515550071) -> (continuation of Deferred@1062375541 after retry RPC after error@3850907) -> (continuation of Deferred@992206199 after retry RPC after error@1888036983) -> (continuation of Deferred@1542704939 after retry RPC after error@227270868) -> (continuation of Deferred@677796983 after retry RPC after error@294982556) -> (continuation of Deferred@1415191570 after retry RPC after error@101042957) -> (continuation of Deferred@471509598 after retry RPC after error@623526834) -> (continuation of Deferred@563372425 after retry RPC after error@165776149), errback=(continuation of Deferred@121946548 after retry RPC after error@574861958) -> (continuation of Deferred@2135465286 after retry RPC after error@515550071) -> (continuation of Deferred@1062375541 after retry RPC after error@3850907) -> (continuation of Deferred@992206199 after retry RPC after error@1888036983) -> (continuation of Deferred@1542704939 after retry RPC after error@227270868) -> (continuation of Deferred@677796983 after retry RPC after error@294982556) -> (continuation of Deferred@1415191570 after retry RPC after error@101042957) -> (continuation of Deferred@471509598 after retry RPC after error@623526834) -> (continuation of Deferred@563372425 after retry RPC after error@165776149)))
03:00:25.982 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=10, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [161ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [162ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [201ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [202ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1690348130(state=PENDING, result=null, callback=(continuation of Deferred@121946548 after retry RPC after error@574861958) -> (continuation of Deferred@2135465286 after retry RPC after error@515550071) -> (continuation of Deferred@1062375541 after retry RPC after error@3850907) -> (continuation of Deferred@992206199 after retry RPC after error@1888036983) -> (continuation of Deferred@1542704939 after retry RPC after error@227270868) -> (continuation of Deferred@677796983 after retry RPC after error@294982556) -> (continuation of Deferred@1415191570 after retry RPC after error@101042957) -> (continuation of Deferred@471509598 after retry RPC after error@623526834) -> (continuation of Deferred@563372425 after retry RPC after error@165776149), errback=(continuation of Deferred@121946548 after retry RPC after error@574861958) -> (continuation of Deferred@2135465286 after retry RPC after error@515550071) -> (continuation of Deferred@1062375541 after retry RPC after error@3850907) -> (continuation of Deferred@992206199 after retry RPC after error@1888036983) -> (continuation of Deferred@1542704939 after retry RPC after error@227270868) -> (continuation of Deferred@677796983 after retry RPC after error@294982556) -> (continuation of Deferred@1415191570 after retry RPC after error@101042957) -> (continuation of Deferred@471509598 after retry RPC after error@623526834) -> (continuation of Deferred@563372425 after retry RPC after error@165776149)))
03:00:25.999 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.000 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.000 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.000 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.000 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.000 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.000 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.019 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.019 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.020 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.020 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.020 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.020 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.020 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.020 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.040 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.041 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.041 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.041 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.041 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.041 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.041 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.060 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.060 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.060 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.060 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.060 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.060 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.060 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.079 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.080 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.080 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.080 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.080 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.080 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.080 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.080 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.081 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.100 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.101 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.101 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.101 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.101 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.101 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.101 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.102 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.140 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.141 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.141 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.141 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.141 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.141 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.141 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.142 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.180 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.180 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.180 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.181 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.181 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.181 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.181 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.181 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.181 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1961035848(state=PENDING, result=null, callback=(continuation of Deferred@1079341168 after retry RPC after error@588560653) -> (continuation of Deferred@569122829 after retry RPC after error@1294060848) -> (continuation of Deferred@639427092 after retry RPC after error@549911307) -> (continuation of Deferred@1617578838 after retry RPC after error@593580582) -> (continuation of Deferred@1647052828 after retry RPC after error@1973791217) -> (continuation of Deferred@1193766120 after retry RPC after error@1853810248) -> (continuation of Deferred@1821405043 after retry RPC after error@874690865), errback=(continuation of Deferred@1079341168 after retry RPC after error@588560653) -> (continuation of Deferred@569122829 after retry RPC after error@1294060848) -> (continuation of Deferred@639427092 after retry RPC after error@549911307) -> (continuation of Deferred@1617578838 after retry RPC after error@593580582) -> (continuation of Deferred@1647052828 after retry RPC after error@1973791217) -> (continuation of Deferred@1193766120 after retry RPC after error@1853810248) -> (continuation of Deferred@1821405043 after retry RPC after error@874690865)))
03:00:26.182 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [103ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@1961035848(state=PENDING, result=null, callback=(continuation of Deferred@1079341168 after retry RPC after error@588560653) -> (continuation of Deferred@569122829 after retry RPC after error@1294060848) -> (continuation of Deferred@639427092 after retry RPC after error@549911307) -> (continuation of Deferred@1617578838 after retry RPC after error@593580582) -> (continuation of Deferred@1647052828 after retry RPC after error@1973791217) -> (continuation of Deferred@1193766120 after retry RPC after error@1853810248) -> (continuation of Deferred@1821405043 after retry RPC after error@874690865), errback=(continuation of Deferred@1079341168 after retry RPC after error@588560653) -> (continuation of Deferred@569122829 after retry RPC after error@1294060848) -> (continuation of Deferred@639427092 after retry RPC after error@549911307) -> (continuation of Deferred@1617578838 after retry RPC after error@593580582) -> (continuation of Deferred@1647052828 after retry RPC after error@1973791217) -> (continuation of Deferred@1193766120 after retry RPC after error@1853810248) -> (continuation of Deferred@1821405043 after retry RPC after error@874690865)))
03:00:26.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.200 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.200 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.200 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.200 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.200 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.200 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.200 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.219 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.219 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.220 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.220 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.220 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.220 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.220 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.220 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.240 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.240 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.240 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.241 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.241 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.241 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.241 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.241 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.260 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.260 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.260 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.260 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.260 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.260 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.261 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.261 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.279 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.279 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.280 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.280 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.280 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.280 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.280 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.280 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.300 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.301 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.301 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.301 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.301 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.301 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.301 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.340 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.341 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.341 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.341 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.341 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.341 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.341 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.342 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.380 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.381 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.381 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.381 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.381 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.381 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.381 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.440 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.441 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.441 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.441 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.441 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.441 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.441 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.442 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@566109746(state=PENDING, result=null, callback=(continuation of Deferred@695811529 after retry RPC after error@1849747689) -> (continuation of Deferred@1588200690 after retry RPC after error@242558627) -> (continuation of Deferred@1845679569 after retry RPC after error@132939608) -> (continuation of Deferred@166688451 after retry RPC after error@780667235) -> (continuation of Deferred@1448934718 after retry RPC after error@850288146) -> (continuation of Deferred@1082490858 after retry RPC after error@774419848) -> (continuation of Deferred@522983550 after retry RPC after error@1605536583) -> (continuation of Deferred@874759540 after retry RPC after error@1598861965), errback=(continuation of Deferred@695811529 after retry RPC after error@1849747689) -> (continuation of Deferred@1588200690 after retry RPC after error@242558627) -> (continuation of Deferred@1845679569 after retry RPC after error@132939608) -> (continuation of Deferred@166688451 after retry RPC after error@780667235) -> (continuation of Deferred@1448934718 after retry RPC after error@850288146) -> (continuation of Deferred@1082490858 after retry RPC after error@774419848) -> (continuation of Deferred@522983550 after retry RPC after error@1605536583) -> (continuation of Deferred@874759540 after retry RPC after error@1598861965)))
03:00:26.442 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [142ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@566109746(state=PENDING, result=null, callback=(continuation of Deferred@695811529 after retry RPC after error@1849747689) -> (continuation of Deferred@1588200690 after retry RPC after error@242558627) -> (continuation of Deferred@1845679569 after retry RPC after error@132939608) -> (continuation of Deferred@166688451 after retry RPC after error@780667235) -> (continuation of Deferred@1448934718 after retry RPC after error@850288146) -> (continuation of Deferred@1082490858 after retry RPC after error@774419848) -> (continuation of Deferred@522983550 after retry RPC after error@1605536583) -> (continuation of Deferred@874759540 after retry RPC after error@1598861965), errback=(continuation of Deferred@695811529 after retry RPC after error@1849747689) -> (continuation of Deferred@1588200690 after retry RPC after error@242558627) -> (continuation of Deferred@1845679569 after retry RPC after error@132939608) -> (continuation of Deferred@166688451 after retry RPC after error@780667235) -> (continuation of Deferred@1448934718 after retry RPC after error@850288146) -> (continuation of Deferred@1082490858 after retry RPC after error@774419848) -> (continuation of Deferred@522983550 after retry RPC after error@1605536583) -> (continuation of Deferred@874759540 after retry RPC after error@1598861965)))
03:00:26.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.460 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.460 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.461 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.461 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.461 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.461 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.461 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.480 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.480 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.480 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.480 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.480 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.480 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.480 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.481 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.499 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.500 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.500 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.500 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.500 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.500 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.500 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.500 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.520 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.521 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.521 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.521 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.521 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.521 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.521 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.522 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.540 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.540 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.541 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.541 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.541 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.541 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.541 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.560 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.560 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.560 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.560 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.560 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.561 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.561 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.562 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.600 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.600 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.600 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.600 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.600 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.600 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.601 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.680 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.680 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.680 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.680 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.680 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.681 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.681 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.681 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@122237638(state=PENDING, result=null, callback=(continuation of Deferred@11085379 after retry RPC after error@910416723) -> (continuation of Deferred@252417113 after retry RPC after error@1100634236) -> (continuation of Deferred@147383602 after retry RPC after error@544816699) -> (continuation of Deferred@1929629489 after retry RPC after error@1615409301) -> (continuation of Deferred@803715291 after retry RPC after error@1183237114) -> (continuation of Deferred@722597468 after retry RPC after error@41268787) -> (continuation of Deferred@281475014 after retry RPC after error@1613721399), errback=(continuation of Deferred@11085379 after retry RPC after error@910416723) -> (continuation of Deferred@252417113 after retry RPC after error@1100634236) -> (continuation of Deferred@147383602 after retry RPC after error@544816699) -> (continuation of Deferred@1929629489 after retry RPC after error@1615409301) -> (continuation of Deferred@803715291 after retry RPC after error@1183237114) -> (continuation of Deferred@722597468 after retry RPC after error@41268787) -> (continuation of Deferred@281475014 after retry RPC after error@1613721399)))
03:00:26.682 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [20ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [40ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [100ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [140ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [220ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [221ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, deferred=Deferred@122237638(state=PENDING, result=null, callback=(continuation of Deferred@11085379 after retry RPC after error@910416723) -> (continuation of Deferred@252417113 after retry RPC after error@1100634236) -> (continuation of Deferred@147383602 after retry RPC after error@544816699) -> (continuation of Deferred@1929629489 after retry RPC after error@1615409301) -> (continuation of Deferred@803715291 after retry RPC after error@1183237114) -> (continuation of Deferred@722597468 after retry RPC after error@41268787) -> (continuation of Deferred@281475014 after retry RPC after error@1613721399), errback=(continuation of Deferred@11085379 after retry RPC after error@910416723) -> (continuation of Deferred@252417113 after retry RPC after error@1100634236) -> (continuation of Deferred@147383602 after retry RPC after error@544816699) -> (continuation of Deferred@1929629489 after retry RPC after error@1615409301) -> (continuation of Deferred@803715291 after retry RPC after error@1183237114) -> (continuation of Deferred@722597468 after retry RPC after error@41268787) -> (continuation of Deferred@281475014 after retry RPC after error@1613721399)))
03:00:26.699 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.700 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.700 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.700 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.700 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.700 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.700 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.700 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.720 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.721 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.721 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.721 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.721 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.721 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.721 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.722 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.740 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.740 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.740 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.740 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.740 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.740 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.740 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.759 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.760 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.760 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.760 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.760 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.760 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.760 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.781 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.781 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.781 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.781 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.781 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.781 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.782 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.800 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.800 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.800 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.800 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.801 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.801 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.801 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.819 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.820 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.820 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.820 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.820 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.820 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.820 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.820 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.879 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.880 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.880 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.880 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.880 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.880 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.880 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.939 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.940 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.940 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.940 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.941 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.941 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.941 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.941 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.941 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1439205742(state=PENDING, result=null, callback=(continuation of Deferred@668877715 after retry RPC after error@266252591) -> (continuation of Deferred@1344784496 after retry RPC after error@1094039779) -> (continuation of Deferred@346208653 after retry RPC after error@1854879231) -> (continuation of Deferred@8717334 after retry RPC after error@1813726789) -> (continuation of Deferred@1844765850 after retry RPC after error@2051212310) -> (continuation of Deferred@1380846332 after retry RPC after error@1683048350) -> (continuation of Deferred@776712122 after retry RPC after error@1288334801), errback=(continuation of Deferred@668877715 after retry RPC after error@266252591) -> (continuation of Deferred@1344784496 after retry RPC after error@1094039779) -> (continuation of Deferred@346208653 after retry RPC after error@1854879231) -> (continuation of Deferred@8717334 after retry RPC after error@1813726789) -> (continuation of Deferred@1844765850 after retry RPC after error@2051212310) -> (continuation of Deferred@1380846332 after retry RPC after error@1683048350) -> (continuation of Deferred@776712122 after retry RPC after error@1288334801)))
03:00:26.942 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [41ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [61ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [62ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [82ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [83ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [101ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [102ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [182ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [242ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, deferred=Deferred@1439205742(state=PENDING, result=null, callback=(continuation of Deferred@668877715 after retry RPC after error@266252591) -> (continuation of Deferred@1344784496 after retry RPC after error@1094039779) -> (continuation of Deferred@346208653 after retry RPC after error@1854879231) -> (continuation of Deferred@8717334 after retry RPC after error@1813726789) -> (continuation of Deferred@1844765850 after retry RPC after error@2051212310) -> (continuation of Deferred@1380846332 after retry RPC after error@1683048350) -> (continuation of Deferred@776712122 after retry RPC after error@1288334801), errback=(continuation of Deferred@668877715 after retry RPC after error@266252591) -> (continuation of Deferred@1344784496 after retry RPC after error@1094039779) -> (continuation of Deferred@346208653 after retry RPC after error@1854879231) -> (continuation of Deferred@8717334 after retry RPC after error@1813726789) -> (continuation of Deferred@1844765850 after retry RPC after error@2051212310) -> (continuation of Deferred@1380846332 after retry RPC after error@1683048350) -> (continuation of Deferred@776712122 after retry RPC after error@1288334801)))
03:00:26.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.960 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.960 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.960 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.960 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.960 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.960 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.961 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:26.979 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:26.979 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:26.980 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.980 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.980 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.980 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.980 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:26.980 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:26.981 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:27.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:27.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:27.000 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:27.000 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:27.001 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:27.001 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:27.001 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:27.001 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:27.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:27.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:27.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:27.021 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:27.021 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:27.021 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:27.021 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:27.021 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:27.021 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:27.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:27.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.254:46111(127.19.203.254:46111)] connecting to peer
03:00:27.040 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.19.203.252:40563(127.19.203.252:40563)] connecting to peer
03:00:27.040 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:27.040 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:27.040 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.252:40563(127.19.203.252:40563)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:27.040 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.19.203.254:46111(127.19.203.254:46111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:27.041 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:27.041 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:27.041 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.254:46111: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111
03:00:27.041 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.203.252:40563: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563
03:00:27.041 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:27.042 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563; will retry
03:00:27.042 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=null, attempt=38, TimeoutTracker(timeout=50000, elapsed=47743), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [44ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [44ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [44ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [64ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] refreshing cache from master, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [84ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [84ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [84ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] refreshing cache from master, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [124ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [144ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [144ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [145ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] refreshing cache from master, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [184ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [184ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [185ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [186ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [186ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [203ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [204ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [204ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [204ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [542ms] refreshing cache from master, [542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [543ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [544ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [544ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [544ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [702ms] refreshing cache from master, [702ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [702ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [703ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [703ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [703ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [704ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2061ms] refreshing cache from master, [2061ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [2061ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [2062ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2063ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2063ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2063ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2063ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2541ms] refreshing cache from master, [2541ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [2542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [2542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2543ms] too many traces: truncated at 100 traces, deferred=Deferred@245725080(state=PENDING, result=null, callback=(continuation of Deferred@482351597 after retry RPC after error@1467449118) -> (continuation of Deferred@1506057862 after retry RPC after error@2088254353) -> (continuation of Deferred@1626036866 after retry RPC after error@210261518) -> (continuation of Deferred@1908791735 after retry RPC after error@1281151149) -> (continuation of Deferred@1985592558 after retry RPC after error@951041037) -> (continuation of Deferred@1735971534 after retry RPC after error@1300999988) -> (continuation of Deferred@1980679335 after retry RPC after error@855916991) -> (continuation of Deferred@315300787 after retry RPC after error@44692689) -> (continuation of Deferred@33694687 after retry RPC after error@328893337) -> (continuation of Deferred@1497765082 after retry RPC after error@335838303) -> (continuation of Deferred@126593499 after retry RPC after error@1087005944) -> (continuation of Deferred@700668159 after retry RPC after error@512900892) -> (continuation of Deferred@1141741668 after retry RPC after error@1467206073) -> (continuation of Deferred@1682204424 after retry RPC after error@1797381699) -> (continuation of Deferred@1128769019 after retry RPC after error@179339819) -> (continuation of Deferred@1301010514 after retry RPC after error@675136614) -> (continuation of Deferred@1454101735 after retry RPC after error@1485092291) -> (continuation of Deferred@1272889289 after retry RPC after error@218520608) -> (continuation of Deferred@92450187 after retry RPC after error@2027235925) -> (continuation of Deferred@1179382385 after retry RPC after error@2031309252) -> (continuation of Deferred@1777157528 after retry RPC after error@951799154) -> (continuation of Deferred@570095726 after retry RPC after error@1115951332) -> (continuation of Deferred@1095126185 after retry RPC after error@603216174) -> (continuation of Deferred@1429994678 after retry RPC after error@937084357) -> (continuation of Deferred@1141568659 after retry RPC after error@1161226261) -> (continuation of Deferred@1331851849 after retry RPC after error@787968653) -> (continuation of Deferred@1028302252 after retry RPC after error@1338934352) -> (continuation of Deferred@333596634 after retry RPC after error@110373937) -> (continuation of Deferred@1847303818 after retry RPC after error@1439464221) -> (continuation of Deferred@1418803945 after retry RPC after error@1933799918) -> (continuation of Deferred@1092505282 after retry RPC after error@2000470383) -> (continuation of Deferred@707263251 after retry RPC after error@1107506104) -> (continuation of Deferred@1909988708 after retry RPC after error@1618910601) -> (continuation of Deferred@1972641270 after retry RPC after error@785477135) -> (continuation of Deferred@525559180 after retry RPC after error@198521146) -> (continuation of Deferred@1593570843 after retry RPC after error@2080332509) -> (continuation of Deferred@2139369259 after retry RPC after error@1576705280), errback=(continuation of Deferred@482351597 after retry RPC after error@1467449118) -> (continuation of Deferred@1506057862 after retry RPC after error@2088254353) -> (continuation of Deferred@1626036866 after retry RPC after error@210261518) -> (continuation of Deferred@1908791735 after retry RPC after error@1281151149) -> (continuation of Deferred@1985592558 after retry RPC after error@951041037) -> (continuation of Deferred@1735971534 after retry RPC after error@1300999988) -> (continuation of Deferred@1980679335 after retry RPC after error@855916991) -> (continuation of Deferred@315300787 after retry RPC after error@44692689) -> (continuation of Deferred@33694687 after retry RPC after error@328893337) -> (continuation of Deferred@1497765082 after retry RPC after error@335838303) -> (continuation of Deferred@126593499 after retry RPC after error@1087005944) -> (continuation of Deferred@700668159 after retry RPC after error@512900892) -> (continuation of Deferred@1141741668 after retry RPC after error@1467206073) -> (continuation of Deferred@1682204424 after retry RPC after error@1797381699) -> (continuation of Deferred@1128769019 after retry RPC after error@179339819) -> (continuation of Deferred@1301010514 after retry RPC after error@675136614) -> (continuation of Deferred@1454101735 after retry RPC after error@1485092291) -> (continuation of Deferred@1272889289 after retry RPC after error@218520608) -> (continuation of Deferred@92450187 after retry RPC after error@2027235925) -> (continuation of Deferred@1179382385 after retry RPC after error@2031309252) -> (continuation of Deferred@1777157528 after retry RPC after error@951799154) -> (continuation of Deferred@570095726 after retry RPC after error@1115951332) -> (continuation of Deferred@1095126185 after retry RPC after error@603216174) -> (continuation of Deferred@1429994678 after retry RPC after error@937084357) -> (continuation of Deferred@1141568659 after retry RPC after error@1161226261) -> (continuation of Deferred@1331851849 after retry RPC after error@787968653) -> (continuation of Deferred@1028302252 after retry RPC after error@1338934352) -> (continuation of Deferred@333596634 after retry RPC after error@110373937) -> (continuation of Deferred@1847303818 after retry RPC after error@1439464221) -> (continuation of Deferred@1418803945 after retry RPC after error@1933799918) -> (continuation of Deferred@1092505282 after retry RPC after error@2000470383) -> (continuation of Deferred@707263251 after retry RPC after error@1107506104) -> (continuation of Deferred@1909988708 after retry RPC after error@1618910601) -> (continuation of Deferred@1972641270 after retry RPC after error@785477135) -> (continuation of Deferred@525559180 after retry RPC after error@198521146) -> (continuation of Deferred@1593570843 after retry RPC after error@2080332509) -> (continuation of Deferred@2139369259 after retry RPC after error@1576705280)))
03:00:27.043 [DEBUG - main] (Connection.java:575) [peer master-127.19.203.253:34409(127.19.203.253:34409)] disconnecting while in state READY
03:00:27.043 [DEBUG - main] (Connection.java:575) [peer master-127.19.203.254:46111(127.19.203.254:46111)] disconnecting while in state TERMINATED
03:00:27.044 [DEBUG - main] (Connection.java:575) [peer master-127.19.203.252:40563(127.19.203.252:40563)] disconnecting while in state TERMINATED
03:00:27.044 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:00:27.045 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.19.203.253:34409(127.19.203.253:34409)] handling channelInactive
03:00:27.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.203.253:34409(127.19.203.253:34409)] cleaning up while in state READY due to: connection closed
03:00:27.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:27.045492 20271 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskKOuND7/build/asan/bin/kudu pid 20579
03:00:27.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:27.059530 20718 generic_service.cc:196] Checking for leaks (request via RPC)
03:00:27.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:27.209430 20271 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskKOuND7/build/asan/bin/kudu with pid 20579
03:00:27.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:27.233474 20271 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskKOuND7/build/asan/bin/kudu pid 20728
03:00:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:27.241076 20859 generic_service.cc:196] Checking for leaks (request via RPC)
03:00:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:27.353193 20271 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskKOuND7/build/asan/bin/kudu with pid 20728
03:00:27.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:27.374552 20271 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskKOuND7/build/asan/bin/kudu pid 20869
03:00:27.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:27.382078 21000 generic_service.cc:196] Checking for leaks (request via RPC)
03:00:27.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:27.500993 20271 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskKOuND7/build/asan/bin/kudu with pid 20869
03:00:27.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:27.521951 20271 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskKOuND7/build/asan/bin/kudu pid 20360
03:00:27.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:27.523238 20434 generic_service.cc:196] Checking for leaks (request via RPC)
03:00:27.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261121 03:00:27.667867 20271 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskKOuND7/build/asan/bin/kudu with pid 20360
03:00:27.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-21T03:00:27Z chronyd exiting
03:00:27.837 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Timed out: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=null, attempt=38, TimeoutTracker(timeout=50000, elapsed=47743), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [44ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [44ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [44ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [64ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] refreshing cache from master, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [84ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [84ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [84ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] refreshing cache from master, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [124ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [144ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [144ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [145ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] refreshing cache from master, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [184ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [184ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [185ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [186ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [186ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [203ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [204ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [204ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [204ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [542ms] refreshing cache from master, [542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [543ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [544ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [544ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [544ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [702ms] refreshing cache from master, [702ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [702ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [703ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [703ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [703ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [704ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2061ms] refreshing cache from master, [2061ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [2061ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [2062ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2063ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2063ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2063ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2063ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2541ms] refreshing cache from master, [2541ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [2542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [2542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2543ms] too many traces: truncated at 100 traces, deferred=Deferred@245725080(state=PENDING, result=null, callback=(continuation of Deferred@482351597 after retry RPC after error@1467449118) -> (continuation of Deferred@1506057862 after retry RPC after error@2088254353) -> (continuation of Deferred@1626036866 after retry RPC after error@210261518) -> (continuation of Deferred@1908791735 after retry RPC after error@1281151149) -> (continuation of Deferred@1985592558 after retry RPC after error@951041037) -> (continuation of Deferred@1735971534 after retry RPC after error@1300999988) -> (continuation of Deferred@1980679335 after retry RPC after error@855916991) -> (continuation of Deferred@315300787 after retry RPC after error@44692689) -> (continuation of Deferred@33694687 after retry RPC after error@328893337) -> (continuation of Deferred@1497765082 after retry RPC after error@335838303) -> (continuation of Deferred@126593499 after retry RPC after error@1087005944) -> (continuation of Deferred@700668159 after retry RPC after error@512900892) -> (continuation of Deferred@1141741668 after retry RPC after error@1467206073) -> (continuation of Deferred@1682204424 after retry RPC after error@1797381699) -> (continuation of Deferred@1128769019 after retry RPC after error@179339819) -> (continuation of Deferred@1301010514 after retry RPC after error@675136614) -> (continuation of Deferred@1454101735 after retry RPC after error@1485092291) -> (continuation of Deferred@1272889289 after retry RPC after error@218520608) -> (continuation of Deferred@92450187 after retry RPC after error@2027235925) -> (continuation of Deferred@1179382385 after retry RPC after error@2031309252) -> (continuation of Deferred@1777157528 after retry RPC after error@951799154) -> (continuation of Deferred@570095726 after retry RPC after error@1115951332) -> (continuation of Deferred@1095126185 after retry RPC after error@603216174) -> (continuation of Deferred@1429994678 after retry RPC after error@937084357) -> (continuation of Deferred@1141568659 after retry RPC after error@1161226261) -> (continuation of Deferred@1331851849 after retry RPC after error@787968653) -> (continuation of Deferred@1028302252 after retry RPC after error@1338934352) -> (continuation of Deferred@333596634 after retry RPC after error@110373937) -> (continuation of Deferred@1847303818 after retry RPC after error@1439464221) -> (continuation of Deferred@1418803945 after retry RPC after error@1933799918) -> (continuation of Deferred@1092505282 after retry RPC after error@2000470383) -> (continuation of Deferred@707263251 after retry RPC after error@1107506104) -> (continuation of Deferred@1909988708 after retry RPC after error@1618910601) -> (continuation of Deferred@1972641270 after retry RPC after error@785477135) -> (continuation of Deferred@525559180 after retry RPC after error@198521146) -> (continuation of Deferred@1593570843 after retry RPC after error@2080332509) -> (continuation of Deferred@2139369259 after retry RPC after error@1576705280), errback=(continuation of Deferred@482351597 after retry RPC after error@1467449118) -> (continuation of Deferred@1506057862 after retry RPC after error@2088254353) -> (continuation of Deferred@1626036866 after retry RPC after error@210261518) -> (continuation of Deferred@1908791735 after retry RPC after error@1281151149) -> (continuation of Deferred@1985592558 after retry RPC after error@951041037) -> (continuation of Deferred@1735971534 after retry RPC after error@1300999988) -> (continuation of Deferred@1980679335 after retry RPC after error@855916991) -> (continuation of Deferred@315300787 after retry RPC after error@44692689) -> (continuation of Deferred@33694687 after retry RPC after error@328893337) -> (continuation of Deferred@1497765082 after retry RPC after error@335838303) -> (continuation of Deferred@126593499 after retry RPC after error@1087005944) -> (continuation of Deferred@700668159 after retry RPC after error@512900892) -> (continuation of Deferred@1141741668 after retry RPC after error@1467206073) -> (continuation of Deferred@1682204424 after retry RPC after error@1797381699) -> (continuation of Deferred@1128769019 after retry RPC after error@179339819) -> (continuation of Deferred@1301010514 after retry RPC after error@675136614) -> (continuation of Deferred@1454101735 after retry RPC after error@1485092291) -> (continuation of Deferred@1272889289 after retry RPC after error@218520608) -> (continuation of Deferred@92450187 after retry RPC after error@2027235925) -> (continuation of Deferred@1179382385 after retry RPC after error@2031309252) -> (continuation of Deferred@1777157528 after retry RPC after error@951799154) -> (continuation of Deferred@570095726 after retry RPC after error@1115951332) -> (continuation of Deferred@1095126185 after retry RPC after error@603216174) -> (continuation of Deferred@1429994678 after retry RPC after error@937084357) -> (continuation of Deferred@1141568659 after retry RPC after error@1161226261) -> (continuation of Deferred@1331851849 after retry RPC after error@787968653) -> (continuation of Deferred@1028302252 after retry RPC after error@1338934352) -> (continuation of Deferred@333596634 after retry RPC after error@110373937) -> (continuation of Deferred@1847303818 after retry RPC after error@1439464221) -> (continuation of Deferred@1418803945 after retry RPC after error@1933799918) -> (continuation of Deferred@1092505282 after retry RPC after error@2000470383) -> (continuation of Deferred@707263251 after retry RPC after error@1107506104) -> (continuation of Deferred@1909988708 after retry RPC after error@1618910601) -> (continuation of Deferred@1972641270 after retry RPC after error@785477135) -> (continuation of Deferred@525559180 after retry RPC after error@198521146) -> (continuation of Deferred@1593570843 after retry RPC after error@2080332509) -> (continuation of Deferred@2139369259 after retry RPC after error@1576705280)))
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.AssertionError: Timed out: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=null, attempt=38, TimeoutTracker(timeout=50000, elapsed=47743), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [23ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [42ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [43ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [44ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [44ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [44ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [63ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [64ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [82ms] refreshing cache from master, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [84ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [84ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [84ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [122ms] refreshing cache from master, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [123ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [124ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [143ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [144ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [144ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [145ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [182ms] refreshing cache from master, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [184ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [184ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [185ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [186ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [186ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [203ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [204ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [204ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [204ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [542ms] refreshing cache from master, [542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [543ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [544ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [544ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [544ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [702ms] refreshing cache from master, [702ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [702ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [703ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [703ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [703ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [704ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2061ms] refreshing cache from master, [2061ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [2061ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [2062ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2063ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.254:46111: Network error: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111, [2063ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.252:40563: Network error: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2063ms] Sub RPC ConnectToMaster: received response from server master-127.19.203.253:34409: OK, [2063ms] delaying RPC due to: Service unavailable: Master config (127.19.203.253:34409,127.19.203.254:46111,127.19.203.252:40563) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.254:46111(127.19.203.254:46111): Connection refused: /127.19.203.254:46111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.203.252:40563(127.19.203.252:40563): Connection refused: /127.19.203.252:40563, [2541ms] refreshing cache from master, [2541ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.253:34409, [2542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.254:46111, [2542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.203.252:40563, [2543ms] too many traces: truncated at 100 traces, deferred=Deferred@245725080(state=PENDING, result=null, callback=(continuation of Deferred@482351597 after retry RPC after error@1467449118) -> (continuation of Deferred@1506057862 after retry RPC after error@2088254353) -> (continuation of Deferred@1626036866 after retry RPC after error@210261518) -> (continuation of Deferred@1908791735 after retry RPC after error@1281151149) -> (continuation of Deferred@1985592558 after retry RPC after error@951041037) -> (continuation of Deferred@1735971534 after retry RPC after error@1300999988) -> (continuation of Deferred@1980679335 after retry RPC after error@855916991) -> (continuation of Deferred@315300787 after retry RPC after error@44692689) -> (continuation of Deferred@33694687 after retry RPC after error@328893337) -> (continuation of Deferred@1497765082 after retry RPC after error@335838303) -> (continuation of Deferred@126593499 after retry RPC after error@1087005944) -> (continuation of Deferred@700668159 after retry RPC after error@512900892) -> (continuation of Deferred@1141741668 after retry RPC after error@1467206073) -> (continuation of Deferred@1682204424 after retry RPC after error@1797381699) -> (continuation of Deferred@1128769019 after retry RPC after error@179339819) -> (continuation of Deferred@1301010514 after retry RPC after error@675136614) -> (continuation of Deferred@1454101735 after retry RPC after error@1485092291) -> (continuation of Deferred@1272889289 after retry RPC after error@218520608) -> (continuation of Deferred@92450187 after retry RPC after error@2027235925) -> (continuation of Deferred@1179382385 after retry RPC after error@2031309252) -> (continuation of Deferred@1777157528 after retry RPC after error@951799154) -> (continuation of Deferred@570095726 after retry RPC after error@1115951332) -> (continuation of Deferred@1095126185 after retry RPC after error@603216174) -> (continuation of Deferred@1429994678 after retry RPC after error@937084357) -> (continuation of Deferred@1141568659 after retry RPC after error@1161226261) -> (continuation of Deferred@1331851849 after retry RPC after error@787968653) -> (continuation of Deferred@1028302252 after retry RPC after error@1338934352) -> (continuation of Deferred@333596634 after retry RPC after error@110373937) -> (continuation of Deferred@1847303818 after retry RPC after error@1439464221) -> (continuation of Deferred@1418803945 after retry RPC after error@1933799918) -> (continuation of Deferred@1092505282 after retry RPC after error@2000470383) -> (continuation of Deferred@707263251 after retry RPC after error@1107506104) -> (continuation of Deferred@1909988708 after retry RPC after error@1618910601) -> (continuation of Deferred@1972641270 after retry RPC after error@785477135) -> (continuation of Deferred@525559180 after retry RPC after error@198521146) -> (continuation of Deferred@1593570843 after retry RPC after error@2080332509) -> (continuation of Deferred@2139369259 after retry RPC after error@1576705280), errback=(continuation of Deferred@482351597 after retry RPC after error@1467449118) -> (continuation of Deferred@1506057862 after retry RPC after error@2088254353) -> (continuation of Deferred@1626036866 after retry RPC after error@210261518) -> (continuation of Deferred@1908791735 after retry RPC after error@1281151149) -> (continuation of Deferred@1985592558 after retry RPC after error@951041037) -> (continuation of Deferred@1735971534 after retry RPC after error@1300999988) -> (continuation of Deferred@1980679335 after retry RPC after error@855916991) -> (continuation of Deferred@315300787 after retry RPC after error@44692689) -> (continuation of Deferred@33694687 after retry RPC after error@328893337) -> (continuation of Deferred@1497765082 after retry RPC after error@335838303) -> (continuation of Deferred@126593499 after retry RPC after error@1087005944) -> (continuation of Deferred@700668159 after retry RPC after error@512900892) -> (continuation of Deferred@1141741668 after retry RPC after error@1467206073) -> (continuation of Deferred@1682204424 after retry RPC after error@1797381699) -> (continuation of Deferred@1128769019 after retry RPC after error@179339819) -> (continuation of Deferred@1301010514 after retry RPC after error@675136614) -> (continuation of Deferred@1454101735 after retry RPC after error@1485092291) -> (continuation of Deferred@1272889289 after retry RPC after error@218520608) -> (continuation of Deferred@92450187 after retry RPC after error@2027235925) -> (continuation of Deferred@1179382385 after retry RPC after error@2031309252) -> (continuation of Deferred@1777157528 after retry RPC after error@951799154) -> (continuation of Deferred@570095726 after retry RPC after error@1115951332) -> (continuation of Deferred@1095126185 after retry RPC after error@603216174) -> (continuation of Deferred@1429994678 after retry RPC after error@937084357) -> (continuation of Deferred@1141568659 after retry RPC after error@1161226261) -> (continuation of Deferred@1331851849 after retry RPC after error@787968653) -> (continuation of Deferred@1028302252 after retry RPC after error@1338934352) -> (continuation of Deferred@333596634 after retry RPC after error@110373937) -> (continuation of Deferred@1847303818 after retry RPC after error@1439464221) -> (continuation of Deferred@1418803945 after retry RPC after error@1933799918) -> (continuation of Deferred@1092505282 after retry RPC after error@2000470383) -> (continuation of Deferred@707263251 after retry RPC after error@1107506104) -> (continuation of Deferred@1909988708 after retry RPC after error@1618910601) -> (continuation of Deferred@1972641270 after retry RPC after error@785477135) -> (continuation of Deferred@525559180 after retry RPC after error@198521146) -> (continuation of Deferred@1593570843 after retry RPC after error@2080332509) -> (continuation of Deferred@2139369259 after retry RPC after error@1576705280)))
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)